diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..a64eb447 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + reviewers: + - ibizaman diff --git a/.github/workflows/auto-merge.yaml b/.github/workflows/auto-merge.yaml index 27f73239..815c6f2e 100644 --- a/.github/workflows/auto-merge.yaml +++ b/.github/workflows/auto-merge.yaml @@ -29,7 +29,7 @@ jobs: - uses: reitermarkus/automerge@v2 with: token: ${{ secrets.GH_TOKEN_FOR_UPDATES }} - merge-method: squash + merge-method: rebase do-not-merge-labels: never-merge required-labels: automerge pull-request: ${{ github.event.inputs.pull-request }} diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 30c6215c..373c549f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,22 +20,28 @@ jobs: path-filter: runs-on: ubuntu-latest outputs: - changed: ${{ steps.filter.outputs.changed }} + changed: ${{ steps.filter.outputs.any_changed }} steps: - name: Checkout repository uses: actions/checkout@v4 - - uses: dorny/paths-filter@v3 + - uses: tj-actions/changed-files@v45 id: filter with: - filters: | - changed: - - 'lib/**' - - 'modules/**' - - '!modules/**/docs/**' - - 'test/**' - - '.github/workflows/build.yaml' + files: | + lib/** + modules/** + !modules/**/docs/** + test/** + .github/workflows/build.yaml + separator: "\n" + + - env: + ALL_CHANGED_FILES: ${{ steps.filter.outputs.all_changed_files }} + run: | + echo $ALL_CHANGED_FILES + build-matrix: needs: [ "path-filter" ] @@ -50,7 +56,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} extra-conf: "system-features = nixos-test benchmark big-parallel kvm" - name: Setup Caching - uses: cachix/cachix-action@v14 + uses: cachix/cachix-action@v15 with: name: selfhostblocks authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' @@ -77,7 +83,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} extra-conf: "system-features = nixos-test benchmark big-parallel kvm" - name: Setup Caching - uses: cachix/cachix-action@v14 + uses: cachix/cachix-action@v15 with: name: selfhostblocks authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' @@ -104,7 +110,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} extra-conf: "system-features = nixos-test benchmark big-parallel kvm" - name: Setup Caching - uses: cachix/cachix-action@v14 + uses: cachix/cachix-action@v15 with: name: selfhostblocks authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml index 5a92bb83..dd27bcab 100644 --- a/.github/workflows/demo.yml +++ b/.github/workflows/demo.yml @@ -50,9 +50,9 @@ jobs: uses: actions/checkout@v4 - name: Install nix - uses: cachix/install-nix-action@v20 + uses: cachix/install-nix-action@v30 - - uses: cachix/cachix-action@v14 + - uses: cachix/cachix-action@v15 with: name: selfhostblocks authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' diff --git a/.github/workflows/lock-update.yaml b/.github/workflows/lock-update.yaml index 575bc3f6..0692473b 100644 --- a/.github/workflows/lock-update.yaml +++ b/.github/workflows/lock-update.yaml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nix uses: DeterminateSystems/nix-installer-action@main with: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 9eed4a13..427c2811 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -29,13 +29,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install nix - uses: cachix/install-nix-action@v20 + uses: cachix/install-nix-action@v30 - name: Setup Caching - uses: cachix/cachix-action@v14 + uses: cachix/cachix-action@v15 with: name: selfhostblocks authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' @@ -57,13 +57,13 @@ jobs: public - name: Setup Pages - uses: actions/configure-pages@v3 + uses: actions/configure-pages@v5 - name: Upload artifact - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@v3 with: path: ./public - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v2 + uses: actions/deploy-pages@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 60c1b7ba..24cdcc46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,47 @@ Template: # Upcoming Release +## New Features + +- Add dashboard for SSL certificates validity + and alert they did not renew on time. + +# v0.2.7 + +## New Features + +- Add dashboard for Nextcloud with PHP-FPM exporter. +- Add voice option to Home-Assistant. + +## User Facing Backwards Compatible Changes + +- Add hostname and domain labels for scraped Prometheus metrics and Loki logs. + +# v0.2.6 + +## New Features + +- Add dashboard for deluge. + +# v0.2.5 + +## Other Changes + +- Fix more modules using backup contract. + +# v0.2.4 + +## Other Changes + +- Fix modules using backup contract. + +# v0.2.3 + ## Breaking Changes - Options `before_backup` and `after_backup` for backup contract have been renamed to `beforeBackup` and `afterBackup`. +- All options using the backup and databasebackup contracts now use the new style. ## Other Changes diff --git a/README.md b/README.md index 0f55c44f..ee537448 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,11 @@ that is not like the other server management tools. > production server, this is really just a one person effort for now and there are most certainly > bugs that I didn't discover yet. +### Flake Module + Self Host Blocks is available as a flake. -To use it in your project, add the following flake input: +Also on [flakehub](https://flakehub.com/flake/ibizaman/selfhostblocks?view=usage). +To use it in your existing project, add the following flake input: ```nix inputs.selfhostblocks.url = "github:ibizaman/selfhostblocks"; @@ -52,7 +55,7 @@ Then, pin it to a release/tag with the following snippet. Updating Self Host Blocks to a new version can be done the same way. ```nix -nix flake lock --override-input selfhostblocks github:ibizaman/selfhostblocks/v0.2.2 +nix flake lock --override-input selfhostblocks github:ibizaman/selfhostblocks/v0.2.7 ``` To get started using Self Host Blocks, @@ -70,6 +73,25 @@ Then, to actually configure services, you can choose which one interests you in Head over to the [matrix channel](https://matrix.to/#/#selfhostblocks:matrix.org) for any remaining question, or just to say hi :) +### Installation From Scratch + +I do recommend for this my sibling project [Skarabox][] +which bootstraps a new server and sets up a few tools: + +- Creating a bootable ISO, installable on an USB key. +- [nixos-anywhere](https://github.com/nix-community/nixos-anywhere) to install NixOS headlessly. +- [disko](https://github.com/nix-community/disko) to format the drives using native ZFS encryption with remote unlocking through ssh. +- [sops-nix](https://github.com/Mic92/sops-nix) to handle secrets. +- [deploy-rs](https://github.com/serokell/deploy-rs) to deploy updates. + +[Skarabox]: https://github.com/ibizaman/skarabox + +### Full Example + +See [full example][] in the manual. + +[full example]: https://shb.skarabox.com/usage.html#usage-complete-example + ## Server Management Self Host Blocks provides a standardized configuration for [some services](https://shb.skarabox.com/services.html) provided by nixpkgs. @@ -77,11 +99,67 @@ The goal is to help spread adoption of self-hosting by providing an opinionated Self Host Blocks takes care of common self-hosting needs: - Backup for all services. +- Automatic creation of ZFS datasets per service. - LDAP and SSO integration for most services. - Monitoring with Grafana and Prometheus stack with provided dashboards. - Automatic reverse proxy and certificate management for HTTPS. - VPN and proxy tunneling services. +### Services + +[Provided services](https://shb.skarabox.com/services.html) are: + +- Nextcloud +- Audiobookshelf +- Deluge + *arr stack +- Forgejo +- Grocy +- Hledger +- Home-Assistant +- Jellyfin +- Nextcloud +- Vaultwarden + +Like explained above, those services all benefit from +out of the box backup, +LDAP and SSO integration, +monitoring with Grafana, +reverse proxy and certificate management +and VPN integration for the *arr suite. + +Some services do not have an entry yet in the manual. +To know options for those, the only way for now +is to go to the [All Options][] section of the manual. + +[All Options]: https://shb.skarabox.com/options.html + +### Blocks + +To provided out of the box common functionality, +the services above use the following [common blocks][]: + +[common blocks]: https://shb.skarabox.com/blocks.html + +- Authelia +- BorgBackup +- Davfs +- LDAP +- Monitoring (Grafana - Prometheus - Loki stack) +- Nginx +- PostgreSQL +- Restic +- Sops +- SSL +- Tinyproxy +- VPN +- ZFS + +Those blocks can be used outside of Self Host Blocks too. + +Some blocks do not have an entry yet in the manual. +To know options for those, the only way for now +is to go to the [All Options][] section of the manual. + ### Unified Interfaces SHB's first goal is to provide unified [building blocks](#available-blocks) diff --git a/VERSION b/VERSION index 373f8c6f..967b33ff 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.3 \ No newline at end of file +0.2.7 \ No newline at end of file diff --git a/docs/default.nix b/docs/default.nix index f66962af..2516c0e9 100644 --- a/docs/default.nix +++ b/docs/default.nix @@ -137,6 +137,11 @@ in stdenv.mkDerivation { '@OPTIONS_JSON@' \ ${individualModuleOptionsDocs [ ../modules/blocks/ssl.nix ]}/share/doc/nixos/options.json + substituteInPlace ./modules/blocks/monitoring/docs/default.md \ + --replace \ + '@OPTIONS_JSON@' \ + ${individualModuleOptionsDocs [ ../modules/blocks/monitoring.nix ]}/share/doc/nixos/options.json + substituteInPlace ./modules/blocks/postgresql/docs/default.md \ --replace \ '@OPTIONS_JSON@' \ diff --git a/docs/redirects.json b/docs/redirects.json index 3d6ae518..8c7b54d5 100644 --- a/docs/redirects.json +++ b/docs/redirects.json @@ -14,6 +14,12 @@ "blocks-monitoring-configuration": [ "blocks-monitoring.html#blocks-monitoring-configuration" ], + "blocks-monitoring-nextcloud-dashboard": [ + "blocks-monitoring.html#blocks-monitoring-nextcloud-dashboard" + ], + "blocks-monitoring-deluge-dashboard": [ + "blocks-monitoring.html#blocks-monitoring-deluge-dashboard" + ], "blocks-monitoring-error-dashboard": [ "blocks-monitoring.html#blocks-monitoring-error-dashboard" ], @@ -23,6 +29,126 @@ "blocks-monitoring-provisioning": [ "blocks-monitoring.html#blocks-monitoring-provisioning" ], + "blocks-monitoring-options": [ + "blocks-monitoring.html#blocks-monitoring-options" + ], + "blocks-monitoring-options-shb.monitoring.adminPassword": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.adminPassword" + ], + "blocks-monitoring-options-shb.monitoring.adminPassword.request": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.adminPassword.request" + ], + "blocks-monitoring-options-shb.monitoring.adminPassword.request.group": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.adminPassword.request.group" + ], + "blocks-monitoring-options-shb.monitoring.adminPassword.request.mode": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.adminPassword.request.mode" + ], + "blocks-monitoring-options-shb.monitoring.adminPassword.request.owner": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.adminPassword.request.owner" + ], + "blocks-monitoring-options-shb.monitoring.adminPassword.request.restartUnits": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.adminPassword.request.restartUnits" + ], + "blocks-monitoring-options-shb.monitoring.adminPassword.result": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.adminPassword.result" + ], + "blocks-monitoring-options-shb.monitoring.adminPassword.result.path": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.adminPassword.result.path" + ], + "blocks-monitoring-options-shb.monitoring.contactPoints": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.contactPoints" + ], + "blocks-monitoring-options-shb.monitoring.debugLog": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.debugLog" + ], + "blocks-monitoring-options-shb.monitoring.domain": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.domain" + ], + "blocks-monitoring-options-shb.monitoring.enable": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.enable" + ], + "blocks-monitoring-options-shb.monitoring.grafanaPort": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.grafanaPort" + ], + "blocks-monitoring-options-shb.monitoring.lokiMajorVersion": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.lokiMajorVersion" + ], + "blocks-monitoring-options-shb.monitoring.lokiPort": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.lokiPort" + ], + "blocks-monitoring-options-shb.monitoring.orgId": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.orgId" + ], + "blocks-monitoring-options-shb.monitoring.prometheusPort": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.prometheusPort" + ], + "blocks-monitoring-options-shb.monitoring.provisionDashboards": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.provisionDashboards" + ], + "blocks-monitoring-options-shb.monitoring.secretKey": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.secretKey" + ], + "blocks-monitoring-options-shb.monitoring.secretKey.request": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.secretKey.request" + ], + "blocks-monitoring-options-shb.monitoring.secretKey.request.group": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.secretKey.request.group" + ], + "blocks-monitoring-options-shb.monitoring.secretKey.request.mode": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.secretKey.request.mode" + ], + "blocks-monitoring-options-shb.monitoring.secretKey.request.owner": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.secretKey.request.owner" + ], + "blocks-monitoring-options-shb.monitoring.secretKey.request.restartUnits": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.secretKey.request.restartUnits" + ], + "blocks-monitoring-options-shb.monitoring.secretKey.result": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.secretKey.result" + ], + "blocks-monitoring-options-shb.monitoring.secretKey.result.path": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.secretKey.result.path" + ], + "blocks-monitoring-options-shb.monitoring.smtp": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.smtp" + ], + "blocks-monitoring-options-shb.monitoring.smtp.from_address": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.smtp.from_address" + ], + "blocks-monitoring-options-shb.monitoring.smtp.from_name": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.smtp.from_name" + ], + "blocks-monitoring-options-shb.monitoring.smtp.host": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.smtp.host" + ], + "blocks-monitoring-options-shb.monitoring.smtp.passwordFile": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.smtp.passwordFile" + ], + "blocks-monitoring-options-shb.monitoring.smtp.port": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.smtp.port" + ], + "blocks-monitoring-options-shb.monitoring.smtp.username": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.smtp.username" + ], + "blocks-monitoring-options-shb.monitoring.ssl": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.ssl" + ], + "blocks-monitoring-options-shb.monitoring.ssl.paths": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.ssl.paths" + ], + "blocks-monitoring-options-shb.monitoring.ssl.paths.cert": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.ssl.paths.cert" + ], + "blocks-monitoring-options-shb.monitoring.ssl.paths.key": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.ssl.paths.key" + ], + "blocks-monitoring-options-shb.monitoring.ssl.systemdService": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.ssl.systemdService" + ], + "blocks-monitoring-options-shb.monitoring.subdomain": [ + "blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.subdomain" + ], "blocks-postgresql": [ "blocks-postgresql.html#blocks-postgresql" ], @@ -1133,8 +1259,8 @@ "services-nextcloudserver-features": [ "services-nextcloud.html#services-nextcloudserver-features" ], - "services-nextcloudserver-maintenance": [ - "services-nextcloud.html#services-nextcloudserver-maintenance" + "services-nextcloudserver-dashboard": [ + "services-nextcloud.html#services-nextcloudserver-dashboard" ], "services-nextcloudserver-options": [ "services-nextcloud.html#services-nextcloudserver-options" @@ -1415,6 +1541,15 @@ "services-nextcloudserver-options-shb.nextcloud.phpFpmPoolSettings": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.phpFpmPoolSettings" ], + "services-nextcloudserver-options-shb.nextcloud.phpFpmPrometheusExporter": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.phpFpmPrometheusExporter" + ], + "services-nextcloudserver-options-shb.nextcloud.phpFpmPrometheusExporter.enable": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.phpFpmPrometheusExporter.enable" + ], + "services-nextcloudserver-options-shb.nextcloud.phpFpmPrometheusExporter.port": [ + "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.phpFpmPrometheusExporter.port" + ], "services-nextcloudserver-options-shb.nextcloud.port": [ "services-nextcloud.html#services-nextcloudserver-options-shb.nextcloud.port" ], @@ -1670,9 +1805,15 @@ "usage": [ "usage.html#usage" ], + "usage-complete-example": [ + "usage.html#usage-complete-example" + ], "usage-example-colmena": [ "usage.html#usage-example-colmena" ], + "usage-example-deployrs": [ + "usage.html#usage-example-deployrs" + ], "usage-example-nixosrebuild": [ "usage.html#usage-example-nixosrebuild" ], diff --git a/docs/usage.md b/docs/usage.md index c8aa5031..bb55e559 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -47,10 +47,12 @@ let src = originPkgs; patches = selfhostblocks.patches.${system}; }; -in - nixpkgs = import nixpkgs' { + + shbNixpkgs = import nixpkgs' { inherit system; }; +in + # ... Use shbNixpkgs ``` Advanced users can if they wish use a version of `nixpkgs` of their choosing but then we cannot @@ -68,9 +70,10 @@ Updating Self Host Blocks to a new version can be done the same way. ### Auto Updates {#usage-flake-autoupdate} -To avoid manually updating the `nixpkgs` version, the [GitHub repository][1] for Self Host Blocks -tries to update the `nixpkgs` input daily, verifying all tests pass before accepting this new -`nixpkgs` version. The setup is explained in [this blog post][2]. +To avoid burden on the maintainers to keep `nixpkgs` input updated with upstream, +the [GitHub repository][1] for Self Host Blocks updates the `nixpkgs` input every couple days, +and verifies all tests pass before automatically merging the new `nixpkgs` version. +The setup is explained in [this blog post][2]. [1]: https://github.com/ibizaman/selfhostblocks [2]: https://blog.tiserbox.com/posts/2023-12-25-automated-flake-lock-update-pull-requests-and-merging.html @@ -86,85 +89,124 @@ The following snippets show how to deploy Self Host Blocks using the standard de }; outputs = { self, selfhostblocks }: { - nixosConfigurations = { - machine = selfhostblocks.inputs.nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - selfhostblocks.nixosModules.${system}.default - ]; + let + system = "x86_64-linux"; + originPkgs = selfhostblocks.inputs.nixpkgs; - # Machine specific configuration goes here. + nixpkgs' = originPkgs.legacyPackages.${system}.applyPatches { + name = "nixpkgs-patched"; + src = originPkgs; + patches = selfhostblocks.patches.${system}; + }; + + shbNixpkgs = import nixpkgs' { + inherit system; + }; + in + nixosConfigurations = { + machine = shbNixpkgs.lib.nixosSystem { + inherit system; + modules = [ + selfhostblocks.nixosModules.${system}.default + ]; + }; }; - }; }; } ``` -The above snippet is very minimal as it assumes you have only one machine to deploy to, so `nixpkgs` -is defined exclusively by the `selfhostblocks.inputs.nixpkgs` input. If some machines are not using -Self Host Blocks, you can do the following: +The above snippet assumes one machine to deploy to, +so `nixpkgs` is defined exclusively by the `selfhostblocks` input. +It is more likely that you have multiple machines, +some not using Self Host Blocks, then you can do the following: ```nix { inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + selfhostblocks.url = "github:ibizaman/selfhostblocks"; }; outputs = { self, selfhostblocks }: { - nixosConfigurations = { - machine1 = nixpkgs.lib.nixosSystem { + let + system = "x86_64-linux"; + originPkgs = selfhostblocks.inputs.nixpkgs; + + nixpkgs' = originPkgs.legacyPackages.${system}.applyPatches { + name = "nixpkgs-patched"; + src = originPkgs; + patches = selfhostblocks.patches.${system}; }; - machine2 = selfhostblocks.inputs.nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - selfhostblocks.nixosModules.${system}.default - ]; + shbNixpkgs = import nixpkgs' { + inherit system; + }; + in + nixosConfigurations = { + machine1 = nixpkgs.lib.nixosSystem { + }; - # Machine specific configuration goes here. + machine2 = shbNixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + selfhostblocks.nixosModules.${system}.default + ]; + }; }; - }; }; } ``` +In the above snippet, `machine1` will use the `nixpkgs` version from your inputs +while `machine2` will use the `nixpkgs` version from `selfhostblocks`. ## Example Deployment With Colmena {#usage-example-colmena} -The following snippets show how to deploy Self Host Blocks using the deployment system [Colmena][3]. +The following snippets show how to deploy Self Host Blocks using the deployment system [Colmena][]. -[3]: https://colmena.cli.rs +[colmena]: https://colmena.cli.rs ```nix { inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; - selfhostblocks.url = "github:ibizaman/selfhostblocks"; }; outputs = { self, selfhostblocks }: { - colmena = - let - system = "x86_64-linux"; - in { + let + system = "x86_64-linux"; + originPkgs = selfhostblocks.inputs.nixpkgs; + + nixpkgs' = originPkgs.legacyPackages.${system}.applyPatches { + name = "nixpkgs-patched"; + src = originPkgs; + patches = selfhostblocks.patches.${system}; + }; + + shbNixpkgs = import nixpkgs' { + inherit system; + }; + in + colmena = { meta = { - nixpkgs = import selfhostblocks.inputs.nixpkgs { inherit system; }; + nixpkgs = shbNixpkgs; }; machine = { selfhostblocks, ... }: { imports = [ selfhostblocks.nixosModules.${system}.default ]; - - # Machine specific configuration goes here. }; }; }; } ``` -The above snippet is very minimal as it assumes you have only one machine to deploy to, so `nixpkgs` -is defined exclusively by the `selfhostblocks` input. It is more likely that you have multiple machines, in this case you can use the `colmena.meta.nodeNixpkgs` option: +The above snippet assumes one machine to deploy to, +so `nixpkgs` is defined exclusively by the `selfhostblocks` input. +It is more likely that you have multiple machines, +some not using Self Host Blocks, +in this case you can use the `colmena.meta.nodeNixpkgs` option: ```nix { @@ -175,14 +217,26 @@ is defined exclusively by the `selfhostblocks` input. It is more likely that you }; outputs = { self, selfhostblocks }: { - colmena = { - let - system = "x86_64-linux"; - in { - meta = + let + system = "x86_64-linux"; + originPkgs = selfhostblocks.inputs.nixpkgs; + + nixpkgs' = originPkgs.legacyPackages.${system}.applyPatches { + name = "nixpkgs-patched"; + src = originPkgs; + patches = selfhostblocks.patches.${system}; + }; + + shbNixpkgs = import nixpkgs' { + inherit system; + }; + in + colmena = { + meta = { nixpkgs = import nixpkgs { inherit system; }; + nodeNixpkgs = { - machine2 = import selfhostblocks.inputs.nixpkgs { inherit system; }; + machine2 = shbNixpkgs; }; }; @@ -195,7 +249,163 @@ is defined exclusively by the `selfhostblocks` input. It is more likely that you # Machine specific configuration goes here. }; - }; + }; + }; +} +``` + +In the above snippet, `machine1` will use the `nixpkgs` version from your inputs +while `machine2` will use the `nixpkgs` version from `selfhostblocks`. + +## Example Deployment with deploy-rs {#usage-example-deployrs} + +The following snippets show how to deploy Self Host Blocks using the deployment system [deploy-rs][]. + +[deploy-rs]: https://github.com/serokell/deploy-rs + +```nix +{ + inputs = { + selfhostblocks.url = "github:ibizaman/selfhostblocks"; + }; + + outputs = { self, selfhostblocks }: { + let + system = "x86_64-linux"; + originPkgs = selfhostblocks.inputs.nixpkgs; + + shbNixpkgs = originPkgs.legacyPackages.${system}.applyPatches { + name = "nixpkgs-patched"; + src = originPkgs; + patches = selfhostblocks.patches.${system}; + }; + + shbPkgs = import shbNixpkgs { inherit system; }; + + deployPkgs = import originPkgs { + inherit system; + overlays = [ + deploy-rs.overlay + (self: super: { + deploy-rs = { + inherit (shbPkgs) deploy-rs; + lib = super.deploy-rs.lib; + }; + }) + ]; + }; + in + nixosModules.machine = { + imports = [ + selfhostblocks.nixosModules.${system}.default + ]; + }; + + nixosConfigurations.machine = shbNixpkgs.lib.nixosSystem { + inherit system; + modules = [ + self.nixosModules.machine + ]; + }; + + deploy.nodes.machine = { + hostname = ...; + sshUser = ...; + sshOpts = [ ... ]; + profiles = { + system = { + user = "root"; + path = deployPkgs.deploy-rs.lib.activate.nixos self.nixosConfigurations.machine; + }; + }; + }; + + # From https://github.com/serokell/deploy-rs?tab=readme-ov-file#overall-usage + checks = builtins.mapAttrs (system: deployLib: deployLib.deployChecks self.deploy) deploy-rs.lib; + }; +} +``` + +The above snippet assumes one machine to deploy to, +so `nixpkgs` is defined exclusively by the `selfhostblocks` input. +It is more likely that you have multiple machines, +some not using Self Host Blocks, +in this case you can do: + +```nix +{ + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + + selfhostblocks.url = "github:ibizaman/selfhostblocks"; + }; + + outputs = { self, selfhostblocks }: { + let + system = "x86_64-linux"; + originPkgs = selfhostblocks.inputs.nixpkgs; + + shbNixpkgs = originPkgs.legacyPackages.${system}.applyPatches { + name = "nixpkgs-patched"; + src = originPkgs; + patches = selfhostblocks.patches.${system}; + }; + + shbPkgs = import shbNixpkgs { inherit system; }; + + deployPkgs = import originPkgs { + inherit system; + overlays = [ + deploy-rs.overlay + (self: super: { + deploy-rs = { + inherit (shbPkgs) deploy-rs; + lib = super.deploy-rs.lib; + }; + }) + ]; + }; + in + nixosModules.machine1 = { + # ... + }; + + nixosModules.machine2 = { + imports = [ + selfhostblocks.nixosModules.${system}.default + ]; + }; + + nixosConfigurations.machine1 = nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + self.nixosModules.machine1 + ]; + }; + + nixosConfigurations.machine2 = shbNixpkgs.lib.nixosSystem { + inherit system; + modules = [ + self.nixosModules.machine2 + ]; + }; + + deploy.nodes.machine1 = { + hostname = ...; + sshUser = ...; + sshOpts = [ ... ]; + profiles = { + system = { + user = "root"; + path = deployPkgs.deploy-rs.lib.activate.nixos self.nixosConfigurations.machine1; + }; + }; + }; + + deploy.nodes.machine2 = # Similar here + + # From https://github.com/serokell/deploy-rs?tab=readme-ov-file#overall-usage + checks = builtins.mapAttrs (system: deployLib: deployLib.deployChecks self.deploy) deploy-rs.lib; }; } ``` @@ -203,6 +413,7 @@ is defined exclusively by the `selfhostblocks` input. It is more likely that you In the above snippet, `machine1` will use the `nixpkgs` version from your inputs while `machine2` will use the `nixpkgs` version from `selfhostblocks`. + ## Secrets with sops-nix {#usage-secrets} This section complements the official [sops-nix](https://github.com/Mic92/sops-nix) guide. @@ -307,3 +518,1018 @@ One way to setup secrets management using `sops-nix`: ``` The above snippet uses the [secrets contract](./contracts-secret.html) and [sops block](./blocks-sops.html) to ease the configuration. + +## Complete Example {#usage-complete-example} + +This is my own config, which is using Self Host Blocks +as well as [Skarabox][], my sibling project used to bootstrap a server. + +[Skarabox]: https://github.com/ibizaman/skarabox + +`flake.nix` + +```nix +{ + description = "Ibizaman's config."; + + inputs = { + selfhostblocks.url = "github:ibizaman/selfhostblocks"; + + skarabox.url = "github:ibizaman/skarabox"; + skarabox.inputs.nixpkgs.follows = "selfhostblocks/nixpkgs"; + + deploy-rs.url = "github:serokell/deploy-rs"; + + sops-nix.url = "github:Mic92/sops-nix"; + }; + + outputs = { self, skarabox, selfhostblocks, sops-nix, deploy-rs }: + let + system = "x86_64-linux"; + originPkgs = selfhostblocks.inputs.nixpkgs; + + shbNixpkgs = originPkgs.legacyPackages.${system}.applyPatches { + name = "nixpkgs-patched"; + src = originPkgs; + patches = selfhostblocks.patches.${system}; + }; + + shbPkgs = import shbNixpkgs { inherit system; }; + + # Taken from https://github.com/serokell/deploy-rs?tab=readme-ov-file#overall-usage + deployPkgs = import originPkgs { + inherit system; + overlays = [ + deploy-rs.overlay + (self: super: { + deploy-rs = { + inherit (shbPkgs) deploy-rs; + lib = super.deploy-rs.lib; + }; + }) + ]; + }; + + domain = # My domain here + in + { + nixosModules.skarabox = { + imports = [ + skarabox.nixosModules.skarabox + selfhostblocks.nixosModules.${system}.default + sops-nix.nixosModules.default + ({ config, ... }: { + skarabox.hostname = "skarabox"; + skarabox.username = "skarabox"; + skarabox.disks.rootDisk = "/dev/nvme0n1"; + # 10% of size SSD. Default value assumes drive size is 1 Tb. + skarabox.disks.rootReservation = "100G"; + skarabox.disks.dataDisk1 = "/dev/sda"; + skarabox.disks.dataDisk2 = "/dev/sdb"; + # 5% of size Hard Drives. Default value assumes drive size is 10 Tb. + skarabox.disks.dataReservation = "500G"; + skarabox.sshAuthorizedKeyFile = ./ssh_skarabox.pub; + skarabox.hostId = builtins.readFile ./hostid; + # Needed to be able to ssh to decrypt the SSD. + boot.initrd.availableKernelModules = [ + "rtw88_8821ce" + "r8169" + ]; + sops.defaultSopsFile = ./secrets.yaml; + sops.age = { + sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; + }; + }) + { + me.domain = domain; + } + + ./configuration.nix + ]; + }; + + # Used with nixos-anywere for installation. + nixosConfigurations.skarabox = shbNixpkgs.lib.nixosSystem { + inherit system; + modules = [ + self.nixosModules.skarabox + { + nix.settings.trusted-public-keys = [ + "selfhostblocks.cachix.org-1:H5h6Uj188DObUJDbEbSAwc377uvcjSFOfpxyCFP7cVs=" + ]; + + nix.settings.substituters = [ + "https://selfhostblocks.cachix.org" + ]; + } + ]; + }; + + # Used with deploy-rs for updates. + deploy.nodes.skarabox = { + hostname = domain; + sshUser = "skarabox"; + sshOpts = [ "-o" "IdentitiesOnly=yes" "-i" "ssh_skarabox" ]; + activationTimeout = 600; + profiles = { + system = { + user = "root"; + path = deployPkgs.deploy-rs.lib.activate.nixos self.nixosConfigurations.skarabox; + }; + }; + }; + # From https://github.com/serokell/deploy-rs?tab=readme-ov-file#overall-usage + checks = builtins.mapAttrs (system: deployLib: deployLib.deployChecks self.deploy) deploy-rs.lib; + }; +} +``` + +`configuration.nix` + +```nix +{ config, lib, pkgs, ... }: +let + inherit (lib) optionals optionalAttrs; + inherit (config.me) domain; + + backupCfg = { enable ? true, shbOpt ? null }: name: let + bck = (if shbOpt != null then shbOpt else config.shb.${name}.backup.request); + in { + shb.restic.instances.${name} = { + request = bck; + settings = { + enable = enable; + + passphrase.result = config.shb.sops.secret."${name}/backup/passphrase".result; + + repository = { + path = "/srv/backup/restic/skarabox/${name}"; + timerConfig = { + OnBootSec = "15min"; + OnUnitActiveSec = "1h"; + RandomizedDelaySec = "7min"; + }; + }; + + retention = { + keep_within = "1d"; + keep_hourly = 24; + keep_daily = 7; + keep_weekly = 4; + keep_monthly = 6; + }; + }; + }; + shb.sops.secret."${name}/backup/passphrase" = { + request = config.shb.restic.instances.${name}.settings.passphrase.request; + }; + shb.sops.secret."${name}/backup/b2_access_key_id" = { + request = config.shb.restic.instances.${name}.settings.passphrase.request; + settings.key = "backup/b2/access_key_id"; + }; + shb.sops.secret."${name}/backup/b2_secret_access_key" = { + request = config.shb.restic.instances.${name}.settings.passphrase.request; + settings.key = "backup/b2/secret_access_key"; + }; + }; +in +{ + options = { + me.domain = lib.mkOption { + type = lib.types.str; + }; + }; + + config = lib.mkMerge [ + { + shb.zfs.defaultPoolName = "root"; + sops.defaultSopsFile = ./secrets.yaml; + } + { + services.openssh.listenAddresses = [ + { + addr = "0.0.0.0"; # Needs to be 0.0.0.0 otherwise it cannot bind on boot. + port = 22; + } + { + addr = "0.0.0.0"; + port = 2222; + } + ]; + networking.firewall.allowedTCPPorts = [ 2222 ]; + } + { + networking.firewall.allowedUDPPorts = [ 53 ]; + services.dnsmasq = { + enable = true; + settings = { + # When switching DNS server, accept old leases from previous server. + dhcp-authoritative = true; + + dhcp-range = "192.168.1.101,192.168.1.150,255.255.255.0,6h"; + + dhcp-host = [ + "aa:bb:cc:dd:ee:ff,skarabox,192.168.1.30,infinite" + ]; + + dhcp-option = [ + "3,192.168.1.1" + ]; + + server = [ + # Stubby + # Also https://wiki.archlinux.org/title/Stubby#Change_port + "127.0.0.1#53000" + "::1#53000" + ]; + + log-queries = false; + + # For stubby + proxy-dnssec = true; + + inherit domain; + no-resolv = true; + bogus-priv = true; + strict-order = true; + # Got issues with bind-interface on startup, needing to restart dnsmasq for it to listen correctly. + # bind-interfaces = true; + # add-cpe-id = 858972; + address = (map (hostname: "/${hostname}.${domain}/192.168.1.30") [ + "authelia" + "ldap" + + "forgejo" + "grafana" + "ha" + "hledger" + "jellyfin" + "n" + "vaultwarden" + + "deluge" + "radarr" + "sonarr" + "jackett" + ]) + ++ (map (hostname: "/${hostname}.${domain}/192.168.1.1") [ + "router" + ]); + }; + }; + + services.stubby = { + enable = true; + # https://github.com/getdnsapi/stubby/blob/develop/stubby.yml.example + settings = pkgs.stubby.passthru.settingsExample // { + listen_addresses = [ + "127.0.0.1@53000" + "0::1@53000" + ]; + # https://dnsprivacy.org/public_resolvers/ + # digest from https://nixos.wiki/wiki/Encrypted_DNS#Stubby + upstream_recursive_servers = [{ + address_data = "9.9.9.9"; + tls_auth_name = "dns.quad9.net"; + tls_pubkey_pinset = [{ + digest = "sha256"; + value = "i2kObfz0qIKCGNWt7MjBUeSrh0Dyjb0/zWINImZES+I="; + }]; + } { + address_data = "149.112.112.112"; + tls_auth_name = "dns.quad9.net"; + tls_pubkey_pinset = [{ + digest = "sha256"; + value = "i2kObfz0qIKCGNWt7MjBUeSrh0Dyjb0/zWINImZES+I="; + }]; + }]; + }; + }; + } + { + services.nginx.enable = true; + shb.nginx.accessLog = true; + networking.firewall.allowedTCPPorts = [ 80 443 ]; + + # Need to wait on auth endpoint to be available otherwise nginx can fail to start. + systemd.services.nginx = { + wants = optionals config.services.dnsmasq.enable [ "dnsmasq.service" ]; + after = optionals config.services.dnsmasq.enable [ "dnsmasq.service" ]; + }; + } + + (optionalAttrs true { + shb.user.acme = { + uid = 995; + gid = 994; + }; + + shb.certs.certs.letsencrypt.${domain} = { + inherit domain; + group = "nginx"; + reloadServices = [ "nginx.service" ]; + adminEmail = "ibizaman@${domain}"; + afterAndWants = optionals config.services.dnsmasq.enable [ "dnsmasq.service" ]; + }; + }) + + (optionalAttrs true { + shb.user.lldap = { + uid = 991; + gid = 989; + }; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "ldap.${domain}" ]; + shb.ldap = { + enable = true; + inherit domain; + subdomain = "ldap"; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + ldapPort = 3890; + webUIListenPort = 17170; + dcdomain = "dc=mydomain,dc=com"; + ldapUserPassword.result = config.shb.sops.secret."ldap/user_password".result; + jwtSecret.result = config.shb.sops.secret."ldap/jwt_secret".result; + # restrictAccessIPRange = "192.168.50.0/24"; + debug = false; + }; + shb.sops.secret."ldap/user_password".request = config.shb.ldap.ldapUserPassword.request; + shb.sops.secret."ldap/jwt_secret".request = config.shb.ldap.jwtSecret.request; + + shb.zfs.datasets."safe/ldap2".path = "/var/lib/private/lldap"; + + }) + (backupCfg { + shbOpt = { + user = "root"; + sourceDirectories = [ + "/var/lib/private/lldap/" + ]; + }; } "ldap") + + (optionalAttrs true { + shb.user.authelia = { + uid = 993; + gid = 992; + }; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "authelia.${domain}" ]; + shb.authelia = { + enable = true; + inherit domain; + subdomain = "authelia"; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + + ldapHostname = "127.0.0.1"; + ldapPort = config.shb.ldap.ldapPort; + dcdomain = config.shb.ldap.dcdomain; + + smtp = { + host = "smtp.eu.mailgun.org"; + port = 587; + username = "postmaster@mg.${domain}"; + from_address = "authelia@${domain}"; + password.result = config.shb.sops.secret."authelia/smtp_password".result; + }; + + secrets = { + jwtSecret.result = config.shb.sops.secret."authelia/jwt_secret".result; + ldapAdminPassword.result = config.shb.sops.secret."authelia/ldap_admin_password".result; + sessionSecret.result = config.shb.sops.secret."authelia/session_secret".result; + storageEncryptionKey.result = config.shb.sops.secret."authelia/storage_encryption_key".result; + identityProvidersOIDCHMACSecret.result = config.shb.sops.secret."authelia/hmac_secret".result; + identityProvidersOIDCIssuerPrivateKey.result = config.shb.sops.secret."authelia/private_key".result; + }; + }; + shb.sops.secret."authelia/jwt_secret".request = config.shb.authelia.secrets.jwtSecret.request; + shb.sops.secret."authelia/ldap_admin_password".request = config.shb.authelia.secrets.ldapAdminPassword.request; + shb.sops.secret."authelia/session_secret".request = config.shb.authelia.secrets.sessionSecret.request; + shb.sops.secret."authelia/storage_encryption_key".request = config.shb.authelia.secrets.storageEncryptionKey.request; + shb.sops.secret."authelia/hmac_secret".request = config.shb.authelia.secrets.identityProvidersOIDCHMACSecret.request; + shb.sops.secret."authelia/private_key".request = config.shb.authelia.secrets.identityProvidersOIDCIssuerPrivateKey.request; + shb.sops.secret."authelia/smtp_password".request = config.shb.authelia.smtp.password.request; + + # Need to wait on auth endpoint to be available otherwise nginx can fail to start. + systemd.services."authelia-${domain}" = { + wants = optionals config.services.dnsmasq.enable [ "dnsmasq.service" "nginx.service" ]; + after = optionals config.services.dnsmasq.enable [ "dnsmasq.service" "nginx.service" ]; + }; + + shb.zfs.datasets."safe/authelia-${domain}" = config.shb.authelia.mount; + shb.zfs.datasets."safe/authelia-redis" = config.shb.authelia.mountRedis; + }) + + (optionalAttrs true { + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "forgejo.${domain}" ]; + shb.forgejo = { + enable = true; + subdomain = "forgejo"; + inherit domain; + ssl = config.shb.certs.certs.letsencrypt."${domain}"; + adminPassword.result = config.shb.sops.secret."forgejo/adminPassword".result; + databasePassword.result = config.shb.sops.secret."forgejo/databasePassword".result; + repositoryRoot = "/srv/projects"; + + smtp = { + host = "smtp.eu.mailgun.org"; + port = 587; + username = "postmaster@mg.${domain}"; + from_address = "authelia@${domain}"; + passwordFile = config.shb.sops.secret."forgejo/smtpPassword".result.path; + }; + + ldap = { + enable = true; + host = "127.0.0.1"; + port = config.shb.ldap.ldapPort; + dcdomain = config.shb.ldap.dcdomain; + adminPassword.result = config.shb.sops.secret."forgejo/ldap_admin_password".result; + }; + + sso = { + enable = true; + endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + clientID = "forgejo"; + + sharedSecret.result = config.shb.sops.secret."forgejo/ssoSecret".result; + sharedSecretForAuthelia.result = config.shb.sops.secret."forgejo/authelia/ssoSecret".result; + }; + }; + services.forgejo.settings.repository.ENABLE_PUSH_CREATE_USER = true; + + # Need to wait on auth endpoint to be available otherwise nginx can fail to start. + systemd.services."forgejo" = { + wants = optionals config.services.dnsmasq.enable [ "dnsmasq.service" "nginx.service" ]; + after = optionals config.services.dnsmasq.enable [ "dnsmasq.service" "nginx.service" ]; + }; + + nix.settings.trusted-users = [ "forgejo" ]; + users.users.forgejo.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + ]; + + shb.sops.secret."forgejo/adminPassword" = { + request = config.shb.forgejo.adminPassword.request; + }; + shb.sops.secret."forgejo/databasePassword" = { + request = config.shb.forgejo.databasePassword.request; + }; + shb.sops.secret."forgejo/smtpPassword".request = { + mode = "0400"; + owner = config.services.forgejo.user; + restartUnits = [ "forgejo.service" ]; + }; + shb.sops.secret."forgejo/ldap_admin_password" = { + request = config.shb.forgejo.ldap.adminPassword.request; + settings.key = "ldap/user_password"; + }; + shb.sops.secret."forgejo/ssoSecret" = { + request = config.shb.forgejo.sso.sharedSecret.request; + }; + shb.sops.secret."forgejo/authelia/ssoSecret" = { + request = config.shb.forgejo.sso.sharedSecretForAuthelia.request; + settings.key = "forgejo/ssoSecret"; + }; + + shb.zfs.datasets."safe/forgejo" = config.shb.forgejo.mount; + }) + (optionalAttrs true (backupCfg {} "forgejo")) + + (optionalAttrs true { + shb.user.vaultwarden = { + uid = 989; + gid = 987; + }; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "vaultwarden.${domain}" ]; + shb.vaultwarden = { + enable = true; + inherit domain; + subdomain = "vaultwarden"; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + port = 8222; + authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + databasePassword.result = config.shb.sops.secret."vaultwarden/db".result; + smtp = { + host = "smtp.eu.mailgun.org"; + port = 587; + username = "postmaster@mg.${domain}"; + from_address = "authelia@${domain}"; + password.result = config.shb.sops.secret."vaultwarden/smtp".result; + }; + }; + shb.sops.secret."vaultwarden/db" = { + request = config.shb.vaultwarden.databasePassword.request; + }; + shb.sops.secret."vaultwarden/smtp" = { + request = config.shb.vaultwarden.smtp.password.request; + }; + + shb.zfs.datasets."safe/vaultwarden" = config.shb.vaultwarden.mount; + shb.zfs.datasets."safe/postgresql".path = "/var/lib/postgresql"; + systemd.services."restic-backups-vaultwarden_s3_s3.us-west-000.backblazeb2.com_skarabox-backup_vaultwarden.service" = { + wants = optionals config.services.dnsmasq.enable [ "dnsmasq.service" ]; + after = optionals config.services.dnsmasq.enable [ "dnsmasq.service" ]; + }; + systemd.services."restic-backups-vaultwarden_srv_backup_restic_skarabox_vaultwarden.service" = { + wants = optionals config.services.dnsmasq.enable [ "dnsmasq.service" ]; + after = optionals config.services.dnsmasq.enable [ "dnsmasq.service" ]; + }; + + }) + (optionalAttrs true (backupCfg {} "vaultwarden")) + + (optionalAttrs true { + shb.zfs.datasets."safe/nextcloud".path = "/var/lib/nextcloud"; + shb.zfs.datasets."safe/redis-nextcloud".path = "/var/lib/redis-nextcloud"; + shb.zfs.datasets."nextcloud" = { + poolName = "zdata"; + path = "/srv/nextcloud/data"; + }; + shb.user.nextcloud.uid = 10020; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "n.${domain}" ]; + shb.nextcloud = { + enable = true; + debug = false; + inherit domain; + subdomain = "n"; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + defaultPhoneRegion = "US"; + + version = 29; + dataDir = "/var/lib/nextcloud"; + adminPass.result = config.shb.sops.secret."nextcloud/adminpass".result; + apps = { + previewgenerator.enable = true; + externalStorage = { + enable = true; + userLocalMount.directory = "/srv/nextcloud/data/$user/files"; + userLocalMount.mountName = "/"; + }; + ldap = { + enable = true; + host = "127.0.0.1"; + port = config.shb.ldap.ldapPort; + dcdomain = config.shb.ldap.dcdomain; + adminName = "admin"; + adminPassword.result = config.shb.sops.secret."nextcloud/ldap_admin_password".result; + userGroup = "nextcloud_user"; + }; + sso = { + enable = true; + endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + clientID = "nextcloud"; + + secret.result = config.shb.sops.secret."nextcloud/sso/secret".result; + secretForAuthelia.result = config.shb.sops.secret."authelia/nextcloud_sso_secret".result; + + fallbackDefaultAuth = false; + }; + }; + extraApps = apps: { + inherit (apps) + bookmarks + calendar + contacts + groupfolders + mail + ; + }; + postgresSettings = { + # From https://pgtune.leopard.in.ua/ with: + + # DB Version: 14 + # OS Type: linux + # DB Type: dw + # Total Memory (RAM): 12 GB + # CPUs num: 4 + # Connections num: + # Data Storage: ssd + + max_connections = "400"; + shared_buffers = "3GB"; + effective_cache_size = "9GB"; + maintenance_work_mem = "768MB"; + checkpoint_completion_target = "0.9"; + wal_buffers = "16MB"; + default_statistics_target = "100"; + random_page_cost = "1.1"; + effective_io_concurrency = "200"; + work_mem = "7864kB"; + huge_pages = "off"; + min_wal_size = "1GB"; + max_wal_size = "4GB"; + max_worker_processes = "4"; + max_parallel_workers_per_gather = "2"; + max_parallel_workers = "4"; + max_parallel_maintenance_workers = "2"; + }; + # Chose static and small number of children to avoid too much I/O strain on hard drives. + phpFpmPoolSettings = { + "pm" = "static"; + "pm.max_children" = 150; + }; + }; + systemd.services.postgresql.serviceConfig.Restart = "always"; + # Secret needed for services.nextcloud.config.adminpassFile. + shb.sops.secret."nextcloud/adminpass" = { + request = config.shb.nextcloud.adminPass.request; + }; + shb.sops.secret."nextcloud/ldap_admin_password" = { + request = config.shb.nextcloud.apps.ldap.adminPassword.request; + settings.key = "ldap/user_password"; + }; + + shb.sops.secret."nextcloud/sso/secret" = { + request = config.shb.nextcloud.apps.sso.secret.request; + }; + shb.sops.secret."authelia/nextcloud_sso_secret" = { + request = config.shb.nextcloud.apps.sso.secretForAuthelia.request; + settings.key = "nextcloud/sso/secret"; + }; + }) + (optionalAttrs true (backupCfg {} "nextcloud")) + (optionalAttrs true (backupCfg { + shbOpt = { + user = "nextcloud"; + sourceDirectories = [ + "/srv/nextcloud/data" + ]; + }; + } "nextcloud-data")) + + (optionalAttrs true { + shb.user.jellyfin = { + uid = 984; + gid = 981; + extraGroups = [ "media" ]; + }; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "jellyfin.${domain}" ]; + shb.jellyfin = { + enable = true; + inherit domain; + subdomain = "jellyfin"; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + ldap = { + enable = true; + host = "127.0.0.1"; + port = config.shb.ldap.ldapPort; + dcdomain = config.shb.ldap.dcdomain; + adminPassword.result = config.shb.sops.secret."jellyfin/ldap_password".result; + userGroup = "jellyfin_user"; + }; + sso = { + enable = true; + endpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + clientID = "jellyfin"; + sharedSecret.result = config.shb.sops.secret."jellyfin/sso_secret".result; + sharedSecretForAuthelia.result = config.shb.sops.secret."jellyfin/authelia/sso_secret".result; + userGroup = "jellyfin_user"; + adminUserGroup = "jellyfin_admin"; + }; + }; + shb.sops.secret."jellyfin/ldap_password" = { + request = config.shb.jellyfin.ldap.adminPassword.request; + settings.key = "ldap/user_password"; + }; + shb.sops.secret."jellyfin/sso_secret" = { + request = config.shb.jellyfin.sso.sharedSecret.request; + }; + shb.sops.secret."jellyfin/authelia/sso_secret" = { + request = config.shb.jellyfin.sso.sharedSecretForAuthelia.request; + settings.key = "jellyfin/sso_secret"; + }; + + shb.zfs.datasets."safe/jellyfin".path = "/var/lib/jellyfin"; + }) + (backupCfg {} "jellyfin") + + (optionalAttrs true { + shb.user.hass = { + uid = 286; + gid = 286; + }; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "ha.${domain}" ]; + shb.home-assistant = { + enable = true; + inherit domain; + subdomain = "ha"; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + + config = { + name = "Skarabox"; + country.source = config.shb.sops.secret."home-assistant/country".result.path; + latitude.source = config.shb.sops.secret."home-assistant/latitude_home".result.path; + longitude.source = config.shb.sops.secret."home-assistant/longitude_home".result.path; + time_zone.source = config.shb.sops.secret."home-assistant/time_zone".result.path; + unit_system = "metric"; + }; + + ldap = { + enable = true; + host = "127.0.0.1"; + port = config.shb.ldap.webUIListenPort; + userGroup = "homeassistant_user"; + }; + }; + shb.sops.secret."home-assistant/country".request = { + mode = "0440"; + owner = "hass"; + group = "hass"; + restartUnits = [ "home-assistant.service" ]; + }; + shb.sops.secret."home-assistant/latitude_home".request = { + mode = "0440"; + owner = "hass"; + group = "hass"; + restartUnits = [ "home-assistant.service" ]; + }; + shb.sops.secret."home-assistant/longitude_home".request = { + mode = "0440"; + owner = "hass"; + group = "hass"; + restartUnits = [ "home-assistant.service" ]; + }; + shb.sops.secret."home-assistant/time_zone".request = { + mode = "0440"; + owner = "hass"; + group = "hass"; + restartUnits = [ "home-assistant.service" ]; + }; + services.home-assistant = { + extraComponents = [ + "accuweather" + "apple_tv" + "asuswrt" + "backup" + "bluetooth" + "cast" + "deluge" + "esphome" + "ibeacon" + "icloud" + "ipp" + "jellyfin" + "kegtron" + "kodi" + "nest" + "nmap_tracker" + "openweathermap" + "oralb" + "overkiz" + "philips_js" + "radarr" + "simplisafe" + "somfy" + "somfy_mylink" + "sonarr" + "sonos" + "subaru" + "tradfri" + "wled" + "zha" + + "assist_pipeline" + "conversation" + "piper" + "wake_word" + "whisper" + "wyoming" + ]; + + # Need to add them manually by enabling advanced mode in user profile + # then adding in Settings > Dashboards > Resources: + # - /local/nixos-lovelace-modules/mini-graph-card-bundle.js + # - /local/nixos-lovelace-modules/mini-media-player-bundle.js + customLovelaceModules = with pkgs.home-assistant-custom-lovelace-modules; [ + mini-graph-card + mini-media-player + ]; + extraPackages = python3Packages: [ + python3Packages.grpcio # Needed for nest + ]; + }; + users.users.hass.extraGroups = [ "dialout" ]; + nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ + "python-nest" + ]; + + shb.zfs.datasets."safe/home-assistant".path = "/var/lib/hass"; + }) + (backupCfg {} "home-assistant") + + (optionalAttrs true { + shb.user.grafana = { + uid = 196; + gid = 986; + }; + shb.user.loki = { + uid = 988; + gid = 985; + }; + shb.user.netdata = { + uid = 987; + gid = 984; + }; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "grafana.${domain}" ]; + shb.monitoring = { + enable = true; + inherit domain; + subdomain = "grafana"; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + + contactPoints = [ "ibizaman@${domain}" ]; + adminPassword.result = config.shb.sops.secret."monitoring/admin_password".result; + secretKey.result = config.shb.sops.secret."monitoring/secret_key".result; + smtp = { + from_address = "grafana@${domain}"; + from_name = "Grafana"; + host = "smtp.mailgun.org"; + port = 587; + username = "postmaster@mg.${domain}"; + passwordFile = config.shb.sops.secret."monitoring/smtp".result.path; + }; + }; + shb.sops.secret."monitoring/smtp".request = { + mode = "0400"; + owner = "grafana"; + group = "grafana"; + restartUnits = [ "grafana.service" ]; + }; + shb.sops.secret."monitoring/admin_password".request = config.shb.monitoring.adminPassword.request; + shb.sops.secret."monitoring/secret_key".request = config.shb.monitoring.secretKey.request; + + services.prometheus.scrapeConfigs = [ + { + job_name = "netdata"; + metrics_path = "/api/v1/allmetrics?format=prometheus&help=yes&source=as-collected"; + static_configs = [ + { + targets = ["192.168.50.168:19999"]; + } + ]; + } + ]; + + shb.zfs.datasets."safe/grafana".path = "/var/lib/grafana"; + shb.zfs.datasets."safe/loki".path = "/var/lib/loki"; + shb.zfs.datasets."safe/netdata".path = "/var/lib/netdata"; + }) + + (optionalAttrs true { + shb.user.hledger.uid = 990; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "hledger.${domain}" ]; + shb.hledger = { + enable = true; + inherit domain; + subdomain = "hledger"; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + localNetworkIPRange = "192.168.1.0/24"; + }; + + shb.zfs.datasets."safe/hledger".path = "/var/lib/hledger"; + }) + (backupCfg {} "hledger") + + (optionalAttrs true { + shb.vpn.nordvpnfr = { + enable = true; + provider = "nordvpn"; + dev = "tun2"; + routingNumber = 11; + remoteServerIP = # One of the servers + authFile = config.shb.sops.secret."nordvpnfr/auth".result.path; + proxyPort = 12001; + }; + shb.sops.secret."nordvpnfr/auth".request = { + mode = "0440"; + restartUnits = [ "openvpn-nordvpnfr.service" ]; + }; + + shb.user.deluge.uid = 10001; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ "deluge.${domain}" ]; + shb.deluge = { + enable = true; + inherit domain; + subdomain = "deluge"; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + + daemonPort = 58846; + daemonListenPorts = [ 6881 6889 ]; + webPort = 8112; + # Some things do not work with the proxy, so instead bind to the interface directly + # proxyPort = config.shb.vpn.nordvpnus.proxyPort; + outgoingInterface = config.shb.vpn.nordvpnfr.dev; + authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + localclientPassword.result = config.shb.sops.secret."deluge/auth/localclient".result; + extraUsers = { + ibizaman.password.source = config.shb.sops.secret."deluge/auth/ibizaman".result.path; + }; + prometheusScraperPassword.result = config.shb.sops.secret."deluge/auth/prometheus".result; + settings = { + downloadLocation = "/srv/downloads"; + }; + extraServiceConfig = { + MemoryHigh = "5G"; + MemoryMax = "6G"; + }; + logLevel = "info"; + }; + shb.sops.secret."deluge/auth/localclient".request = config.shb.deluge.localclientPassword.request; + shb.sops.secret."deluge/auth/ibizaman".request = { + mode = "0440"; + owner = config.services.deluge.user; + group = config.services.deluge.group; + restartUnits = [ "deluged.service" "delugeweb.service" ]; + }; + shb.sops.secret."deluge/auth/prometheus".request = config.shb.deluge.prometheusScraperPassword.request; + + shb.zfs.datasets."safe/deluge".path = "/var/lib/deluge"; + }) + (backupCfg {} "deluge") + + (optionalAttrs true { + shb.user.radarr.uid = 10010; + shb.user.radarr.extraGroups = [ "media" ]; + shb.user.sonarr.uid = 10011; + shb.user.sonarr.extraGroups = [ "media" ]; + shb.user.jackett.uid = 10015; + + shb.certs.certs.letsencrypt.${domain}.extraDomains = [ + "moviesdl.${domain}" + "seriesdl.${domain}" + "subtitlesdl.${domain}" + "booksdl.${domain}" + "musicdl.${domain}" + "indexer.${domain}" + ]; + shb.arr = { + radarr = { + subdomain = "radarr"; + inherit domain; + enable = true; + ssl = config.shb.certs.certs.letsencrypt.${domain}; + authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + settings = { + ApiKey.source = config.shb.sops.secret."radarr/apikey".result.path; + }; + }; + sonarr = { + subdomain = "sonarr"; + inherit domain; + enable = true; + ssl = config.shb.certs.certs.letsencrypt."${domain}"; + authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + settings = { + ApiKey.source = config.shb.sops.secret."sonarr/apikey".result.path; + }; + }; + jackett = { + subdomain = "jackett"; + inherit domain; + enable = true; + ssl = config.shb.certs.certs.letsencrypt."${domain}"; + authEndpoint = "https://${config.shb.authelia.subdomain}.${config.shb.authelia.domain}"; + settings = { + ApiKey.source = config.shb.sops.secret."jackett/apikey".result.path; + ProxyType = "0"; + ProxyUrl = "127.0.0.1:${toString config.shb.vpn.nordvpnfr.proxyPort}"; + }; + }; + }; + shb.sops.secret."radarr/apikey".request = { + mode = "0440"; + owner = "radarr"; + group = "radarr"; + restartUnits = [ "radarr.service" ]; + }; + shb.sops.secret."sonarr/apikey".request = { + mode = "0440"; + owner = "sonarr"; + group = "sonarr"; + restartUnits = [ "sonarr.service" ]; + }; + shb.sops.secret."jackett/apikey".request = { + mode = "0440"; + owner = "jackett"; + group = "jackett"; + restartUnits = [ "jackett.service" ]; + }; + + shb.zfs.datasets."safe/radarr".path = "/var/lib/radarr"; + shb.zfs.datasets."safe/sonarr".path = "/var/lib/sonarr"; + shb.zfs.datasets."safe/jackett".path = "/var/lib/jackett"; + }) + (backupCfg { shbOpt = config.shb.arr.radarr; } "radarr") + (backupCfg { shbOpt = config.shb.arr.sonarr; } "sonarr") + (backupCfg { shbOpt = config.shb.arr.jackett; } "jackett") + ]; +} +``` diff --git a/flake.lock b/flake.lock index 5beb12b4..57df81e7 100644 --- a/flake.lock +++ b/flake.lock @@ -35,11 +35,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1732014248, - "narHash": "sha256-y/MEyuJ5oBWrWAic/14LaIr/u5E0wRVzyYsouYY3W6w=", + "lastModified": 1735291276, + "narHash": "sha256-NYVcA06+blsLG6wpAbSPTCyLvxD/92Hy4vlY9WxFI1M=", "owner": "nixos", "repo": "nixpkgs", - "rev": "23e89b7da85c3640bbc2173fe04f4bd114342367", + "rev": "634fd46801442d760e09493a794c4f15db2d0cbb", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 01e0316c..bdfb8897 100644 --- a/flake.nix +++ b/flake.nix @@ -14,12 +14,16 @@ outputs = { nixpkgs, nix-flake-tests, flake-utils, nmdsrc, ... }: flake-utils.lib.eachDefaultSystem (system: let originPkgs = nixpkgs.legacyPackages.${system}; - patches = [ + patches = originPkgs.lib.optionals (system == "x86_64-linux") [ # Leaving commented out for an example. # (originPkgs.fetchpatch { # url = "https://github.com/NixOS/nixpkgs/pull/317107.patch"; # hash = "sha256-hoLrqV7XtR1hP/m0rV9hjYUBtrSjay0qcPUYlKKuVWk="; # }) + + # Remove when this PR is merged: + # https://github.com/NixOS/nixpkgs/pull/368325 + ./patches/prometheusnodecertexporter.nix ]; patchedNixpkgs = originPkgs.applyPatches { name = "nixpkgs-patched"; @@ -28,6 +32,15 @@ }; pkgs = import patchedNixpkgs { inherit system; + + config = { + permittedInsecurePackages = [ + # https://github.com/NixOS/nixpkgs/issues/360592 + "aspnetcore-runtime-6.0.36" + # TODO: https://github.com/NixOS/nixpkgs/issues/326335 + "dotnet-sdk-6.0.428" + ]; + }; }; allModules = [ @@ -124,6 +137,7 @@ // (vm_test "deluge" ./test/services/deluge.nix) // (vm_test "forgejo" ./test/services/forgejo.nix) // (vm_test "grocy" ./test/services/grocy.nix) + // (vm_test "hledger" ./test/services/hledger.nix) // (vm_test "homeassistant" ./test/services/home-assistant.nix) // (vm_test "jellyfin" ./test/services/jellyfin.nix) // (vm_test "monitoring" ./test/services/monitoring.nix) diff --git a/modules/blocks/authelia.nix b/modules/blocks/authelia.nix index 06eaa51b..5d068fa6 100644 --- a/modules/blocks/authelia.nix +++ b/modules/blocks/authelia.nix @@ -525,6 +525,10 @@ in static_configs = [ { targets = ["127.0.0.1:9959"]; + labels = { + "hostname" = config.networking.hostName; + "domain" = cfg.domain; + }; } ]; } diff --git a/modules/blocks/monitoring.nix b/modules/blocks/monitoring.nix index ed34e7d9..c9c21cc9 100644 --- a/modules/blocks/monitoring.nix +++ b/modules/blocks/monitoring.nix @@ -6,6 +6,11 @@ let contracts = pkgs.callPackage ../contracts {}; fqdn = "${cfg.subdomain}.${cfg.domain}"; + + commonLabels = { + hostname = config.networking.hostName; + domain = cfg.domain; + }; in { options.shb.monitoring = { @@ -406,6 +411,8 @@ in path = "/var/log/journal"; # matches = "_TRANSPORT=kernel"; labels = { + domain = cfg.domain; + hostname = config.networking.hostName; job = "systemd-journal"; }; }; @@ -443,6 +450,7 @@ in job_name = "node"; static_configs = [{ targets = ["127.0.0.1:${toString config.services.prometheus.exporters.node.port}"]; + labels = commonLabels; }]; } { @@ -452,24 +460,28 @@ in honor_labels = true; static_configs = [{ targets = [ "127.0.0.1:19999" ]; + labels = commonLabels; }]; } { job_name = "smartctl"; static_configs = [{ targets = ["127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"]; + labels = commonLabels; }]; } { job_name = "prometheus_internal"; static_configs = [{ targets = ["127.0.0.1:${toString config.services.prometheus.port}"]; + labels = commonLabels; }]; } ] ++ (lib.lists.optional config.services.nginx.enable { job_name = "nginx"; static_configs = [{ targets = ["127.0.0.1:${toString config.services.prometheus.exporters.nginx.port}"]; + labels = commonLabels; }]; # }) ++ (lib.optional (builtins.length (lib.attrNames config.services.redis.servers) > 0) { # job_name = "redis"; @@ -489,6 +501,7 @@ in job_name = "dnsmasq"; static_configs = [{ targets = ["127.0.0.1:${toString config.services.prometheus.exporters.dnsmasq.port}"]; + labels = commonLabels; }]; }); services.prometheus.exporters.nginx = lib.mkIf config.services.nginx.enable { diff --git a/modules/blocks/monitoring/dashboards/Errors.json b/modules/blocks/monitoring/dashboards/Errors.json index 132440cf..22f43406 100644 --- a/modules/blocks/monitoring/dashboards/Errors.json +++ b/modules/blocks/monitoring/dashboards/Errors.json @@ -20,7 +20,6 @@ "graphTooltip": 0, "id": 8, "links": [], - "liveNow": false, "panels": [ { "datasource": { @@ -39,6 +38,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -110,7 +110,7 @@ "sort": "none" } }, - "pluginVersion": "10.2.0", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -182,6 +182,7 @@ "axisPlacement": "auto", "axisSoftMin": 0.5, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -254,7 +255,7 @@ "sort": "none" } }, - "pluginVersion": "10.2.0", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -325,6 +326,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -398,7 +400,7 @@ "sort": "none" } }, - "pluginVersion": "10.2.0", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -457,6 +459,10 @@ "type": "loki", "uid": "cd6cc53e-840c-484d-85f7-96fede324006" }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 8, "w": 24, @@ -474,6 +480,7 @@ "sortOrder": "Descending", "wrapLogMessage": false }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -512,8 +519,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -743,28 +749,23 @@ "type": "table" } ], + "preload": false, "refresh": "", - "schemaVersion": 38, + "schemaVersion": 40, "tags": [], "templating": { "list": [ { "allValue": ".+", "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] + "text": "All", + "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "definition": "query_result(max by (name) (node_systemd_unit_state))", - "hide": 0, "includeAll": true, "multi": true, "name": "service", @@ -776,27 +777,23 @@ }, "refresh": 1, "regex": "/name=\"(?.*)\\.service\"/", - "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { - "selected": false, - "text": "", - "value": "" + "text": ".+", + "value": ".+" }, - "hide": 0, "name": "server_name", "options": [ { "selected": true, - "text": "", - "value": "" + "text": ".+", + "value": ".+" } ], "query": ".+", - "skipUrlSync": false, "type": "textbox" } ] @@ -809,6 +806,6 @@ "timezone": "", "title": "Errors", "uid": "d66242cf-71e8-417c-8ef7-51b0741545df", - "version": 29, + "version": 32, "weekStart": "" } diff --git a/modules/blocks/monitoring/dashboards/Nextcloud.json b/modules/blocks/monitoring/dashboards/Nextcloud.json new file mode 100644 index 00000000..515691a6 --- /dev/null +++ b/modules/blocks/monitoring/dashboards/Nextcloud.json @@ -0,0 +1,2178 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 13, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 0, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 19, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.3.0+security-01", + "repeat": "other_service", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{unit=\"$other_service.service\"} | json | line_format \"{{.message}}\" | json | drop message | line_format \"[{{.app}} - {{.url}}] {{.Message}}: {{.exception_details}}{{.Previous_Message}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Panel Title", + "type": "logs" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "description": "Some stall time means that 100% of the CPU is used. It's not an issue if this happens occasionally but can mean the CPU is underpowered for the current use case if this happens most of the time.\nTo fix this, the \"nice\" property of processes can be adjusted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "transparent", + "value": 0.05 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.fillOpacity", + "value": 34 + }, + { + "id": "min", + "value": -40 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "netdata_system_cpu_some_pressure_stall_time_ms_average{hostname=~\"$hostname\"} * -1", + "hide": false, + "instant": false, + "legendFormat": "some stall time", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(dimension, service_name) (netdata_systemd_service_cpu_utilization_percentage_average{hostname=~\"$hostname\", service_name=~\".*$service.*\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service_name}} / {{dimension}}", + "range": true, + "refId": "used", + "useBackend": false + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": -100, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "transparent", + "value": 0.05 + } + ] + }, + "unit": "mbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "decimals" + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.fillOpacity", + "value": 10 + }, + { + "id": "custom.axisPlacement", + "value": "auto" + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "netdata_system_memory_full_pressure_stall_time_ms_average{hostname=~\"$hostname\"} * -1", + "hide": false, + "instant": false, + "legendFormat": "full stall time", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "sum(netdata_mem_available_MiB_average{hostname=~\"$hostname\"})", + "hide": false, + "instant": false, + "legendFormat": "total available", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(dimension, service_name) (netdata_systemd_service_memory_usage_MiB_average{hostname=~\"$hostname\", service_name=~\".*$service.*\", dimension=\"ram\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service_name}}", + "range": true, + "refId": "used", + "useBackend": false + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "transparent", + "value": 0.05 + } + ] + }, + "unit": "KBs" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.fillOpacity", + "value": 34 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(dimension) (netdata_system_net_kilobits_persec_average{hostname=~\"$hostname\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{dimension}}", + "range": true, + "refId": "used", + "useBackend": false + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "transparent", + "value": 0.05 + } + ] + }, + "unit": "Kibits" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.fillOpacity", + "value": 12 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + }, + { + "id": "min", + "value": -200 + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 10 + ], + "fill": "dot" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.fillOpacity", + "value": 17 + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + }, + { + "id": "min", + "value": -200 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "netdata_system_io_full_pressure_stall_time_ms_average{hostname=~\"$hostname\"} * -1", + "hide": false, + "instant": false, + "legendFormat": "full stall time", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "netdata_system_io_some_pressure_stall_time_ms_average{hostname=~\"$hostname\"} * -1", + "hide": false, + "instant": false, + "legendFormat": "some stall time", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(dimension, service_name) (netdata_systemd_service_disk_io_KiB_persec_average{hostname=~\"$hostname\", service_name=~\".*$service.*\", dimension=\"read\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "{{service_name}} / {{dimension}}", + "range": true, + "refId": "read", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "sum by(dimension, service_name) (netdata_systemd_service_disk_io_KiB_persec_average{hostname=~\"$hostname\", service_name=~\".*$service.*\", dimension=\"write\"}) * -1", + "hide": false, + "instant": false, + "legendFormat": "{{service_name}} / {{dimension}}", + "range": true, + "refId": "write" + } + ], + "title": "Disk I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + }, + { + "color": "transparent", + "value": 100 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total" + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum (phpfpm_active_processes{hostname=~\"$hostname\",pool=\"nextcloud\"})", + "hide": false, + "legendFormat": "active", + "range": true, + "refId": "active" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "sum (phpfpm_total_processes{hostname=~\"$hostname\",pool=\"nextcloud\"})", + "hide": false, + "instant": false, + "legendFormat": "total", + "range": true, + "refId": "total" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "sum (phpfpm_max_active_processes{hostname=~\"$hostname\",pool=\"nextcloud\"})", + "hide": false, + "instant": false, + "legendFormat": "max active", + "range": true, + "refId": "max active" + } + ], + "title": "PHP-FPM Processes", + "transformations": [ + { + "disabled": true, + "id": "calculateField", + "options": { + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "total" + } + }, + "operator": "*", + "right": { + "fixed": "0.8" + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 14, + "panels": [], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "description": "If requests occasionally take longer than the threshold time, that's fine. If instead most of the queries take longer than the threshold, performance issue should be investigated.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 700000 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "disableTextWrap": false, + "editorMode": "code", + "expr": "max by (hostname,child) (phpfpm_process_request_duration and (abs(phpfpm_process_request_duration - phpfpm_process_request_duration offset $__interval) > 1))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "PHP-FPM Request Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "rate(phpfpm_max_listen_queue[2m])", + "hide": false, + "instant": false, + "legendFormat": "{{hostname}} - max queue", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "phpfpm_listen_queue", + "legendFormat": "{{hostname}} - queue", + "range": true, + "refId": "A" + } + ], + "title": "PHP-FPM Requests Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "variance" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=\"nginx.service\"} | pattern \"<_> <_> \" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | server_name =~ \"^$subdomain.*\"", + "legendFormat": "", + "queryType": "range", + "refId": "A" + } + ], + "title": "Requests", + "transformations": [ + { + "id": "extractFields", + "options": { + "keepTime": true, + "replace": true, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "body_bytes_sent": true, + "bytes_sent": true, + "gzip_ration": true, + "job": true, + "line": true, + "post": true, + "referrer": true, + "remote_addr": false, + "remote_user": true, + "request": true, + "request_length": true, + "status": true, + "time_local": true, + "unit": true, + "upstream_addr": true, + "upstream_connect_time": true, + "upstream_header_time": true, + "upstream_response_time": true, + "upstream_status": false, + "user_agent": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "", + "destinationType": "number", + "targetField": "request_time" + } + ], + "fields": {} + } + }, + { + "id": "partitionByValues", + "options": { + "fields": [ + "server_name", + "remote_addr" + ], + "keepFields": false + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "request_time (.*)", + "renamePattern": "$1" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "custom.width", + "value": 70 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Time" + } + ] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=\"nginx.service\"} | pattern \"<_> <_> \" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | upstream_addr =~ \"$upstream_addr\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Requests Details", + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "Line" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "id": true, + "labels": true, + "server_name": true, + "time_local": true, + "tsNs": true, + "upstream_addr": true + }, + "includeByName": {}, + "indexByName": { + "Line": 2, + "Time": 1, + "body_bytes_sent": 13, + "bytes_sent": 12, + "gzip_ration": 16, + "id": 4, + "labels": 0, + "post": 17, + "referrer": 14, + "remote_addr": 7, + "remote_user": 8, + "request": 6, + "request_length": 10, + "request_time": 20, + "server_name": 11, + "status": 5, + "time_local": 9, + "tsNs": 3, + "upstream_addr": 18, + "upstream_connect_time": 22, + "upstream_header_time": 23, + "upstream_response_time": 21, + "upstream_status": 19, + "user_agent": 15 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "custom.width", + "value": 70 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 11, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=\"nginx.service\"} | pattern \"<_> <_> \" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | upstream_addr = \"$upstream_addr\" | status =~\"5..\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "5XX Requests Details", + "transformations": [ + { + "id": "extractFields", + "options": { + "source": "Line" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "id": true, + "labels": true, + "server_name": true, + "time_local": true, + "tsNs": true, + "upstream_addr": true + }, + "includeByName": {}, + "indexByName": { + "Line": 2, + "Time": 1, + "body_bytes_sent": 13, + "bytes_sent": 12, + "gzip_ration": 16, + "id": 4, + "labels": 0, + "post": 17, + "referrer": 14, + "remote_addr": 7, + "remote_user": 8, + "request": 6, + "request_length": 10, + "request_time": 20, + "server_name": 11, + "status": 5, + "time_local": 9, + "tsNs": 3, + "upstream_addr": 18, + "upstream_connect_time": 22, + "upstream_header_time": 23, + "upstream_response_time": 21, + "upstream_status": 19, + "user_agent": 15 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 18, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 20, + "maxPerRow": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.4.0", + "repeat": "other_service", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=\"$other_service.service\"} | line_format \"{{ if hasPrefix \\\"{\\\" __line__ }}{{ with $parsed := fromJson __line__ }}[{{.app}} - {{.url}}] {{ if hasPrefix \\\"{\\\" .message }}{{ with $mParsed := fromJson .message }}{{ $mParsed.Message }} @ {{ $mParsed.File }}:{{ $mParsed.Line }}{{ end }}{{ else }}{{ .message }}{{ end }}{{ end }}{{ else }}{{ __line__ }}{{ end }}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log: $other_service", + "type": "logs" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 89 + }, + "id": 15, + "panels": [], + "title": "Backup", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 90 + }, + "id": 16, + "maxPerRow": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.4.0", + "repeat": "service_backup", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=\"$service_backup.service\"}", + "legendFormat": "", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log: $service_backup", + "transformations": [ + { + "disabled": true, + "id": "extractFields", + "options": { + "source": "Line" + } + }, + { + "disabled": true, + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "id": true, + "labels": true, + "tsNs": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "logs" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 100 + }, + "id": 13, + "panels": [], + "title": "Supporting Services", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration_ms" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unit" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "statement" + }, + "properties": [ + { + "id": "custom.width", + "value": 505 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 10, + "links": [ + { + "title": "explore", + "url": "https://grafana.tiserbox.com/explore?panes=%7B%22HWt%22:%7B%22datasource%22:%22cd6cc53e-840c-484d-85f7-96fede324006%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bunit%3D%5C%22nginx.service%5C%22%7D%20%7C%20pattern%20%5C%22%3C_%3E%20%3C_%3E%20%3Cline%3E%5C%22%20%7C%20line_format%20%5C%22%7B%7B.line%7D%7D%5C%22%20%7C%20json%20%7C%20status%20%21~%20%5C%222..%5C%22%20%7C%20__error__%20%21%3D%20%5C%22JSONParserErr%5C%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22cd6cc53e-840c-484d-85f7-96fede324006%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Time" + } + ] + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=\"postgresql.service\"} | regexp \".*duration: (?P[0-9.]+) ms (?P.*)\" | duration_ms > 1000", + "queryType": "range", + "refId": "A" + } + ], + "title": "Slow PostgreSQL Queries", + "transformations": [ + { + "id": "extractFields", + "options": { + "keepTime": false, + "replace": false, + "source": "labels" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Line": true, + "Time": false, + "id": true, + "job": true, + "labels": true, + "tsNs": true, + "unit": true + }, + "includeByName": {}, + "indexByName": { + "Line": 6, + "Time": 0, + "duration_ms": 1, + "id": 8, + "job": 2, + "labels": 5, + "statement": 4, + "tsNs": 7, + "unit": 3 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 109 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{unit=\"redis-$service.service\"} |= ``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Redis", + "type": "logs" + } + ], + "preload": false, + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "baryum" + ], + "value": [ + "baryum" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "definition": "label_values(up,hostname)", + "includeAll": false, + "multi": true, + "name": "hostname", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(up,hostname)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "nextcloud", + "value": "nextcloud" + }, + "description": "", + "hide": 2, + "name": "service", + "query": "nextcloud", + "skipUrlSync": true, + "type": "constant" + }, + { + "current": { + "text": "n", + "value": "n" + }, + "hide": 2, + "name": "subdomain", + "query": "n", + "skipUrlSync": true, + "type": "constant" + }, + { + "current": { + "text": "unix:/run/phpfpm/nextcloud.sock", + "value": "unix:/run/phpfpm/nextcloud.sock" + }, + "hide": 2, + "name": "upstream_addr", + "query": "unix:/run/phpfpm/nextcloud.sock", + "skipUrlSync": true, + "type": "constant" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "definition": "label_values({unit_name=~\".*$service.*\", unit_name=~\".*backups.*\", unit_name!~\".*restore_gen\"},unit_name)", + "description": "", + "hide": 2, + "includeAll": true, + "name": "service_backup", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({unit_name=~\".*$service.*\", unit_name=~\".*backups.*\", unit_name!~\".*restore_gen\"},unit_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(netdata_systemd_service_unit_state_state_average{unit_name=~\".*nextcloud.*\", unit_name!~\".*backup.*\", unit_name!~\".*redis.*\"},unit_name)", + "hide": 2, + "includeAll": true, + "name": "other_service", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(netdata_systemd_service_unit_state_state_average{unit_name=~\".*nextcloud.*\", unit_name!~\".*backup.*\", unit_name!~\".*redis.*\"},unit_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Nextcloud", + "uid": "cdsszybv2gow0d", + "version": 49, + "weekStart": "" +} diff --git a/modules/blocks/monitoring/dashboards/Performance.json b/modules/blocks/monitoring/dashboards/Performance.json index 17cc7a00..bb8c88f2 100644 --- a/modules/blocks/monitoring/dashboards/Performance.json +++ b/modules/blocks/monitoring/dashboards/Performance.json @@ -20,9 +20,9 @@ "graphTooltip": 0, "id": 6, "links": [], - "liveNow": false, "panels": [ { + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -30,6 +30,7 @@ "y": 0 }, "id": 12, + "panels": [], "title": "Node", "type": "row" }, @@ -51,6 +52,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -204,6 +206,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -211,7 +214,7 @@ "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", - "expr": "netdata_system_memory_full_pressure_stall_time_ms_average{instance=~\"$instance\"} * -1", + "expr": "netdata_system_memory_full_pressure_stall_time_ms_average{hostname=~\"$hostname\"} * -1", "hide": false, "instant": false, "legendFormat": "full stall time", @@ -269,6 +272,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -422,6 +426,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -429,7 +434,7 @@ "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", - "expr": "netdata_system_memory_full_pressure_stall_time_ms_average{instance=~\"$instance\"} * -1", + "expr": "netdata_system_memory_full_pressure_stall_time_ms_average{hostname=~\"$hostname\"} * -1", "hide": false, "instant": false, "legendFormat": "full stall time", @@ -456,7 +461,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(dimension, service_name) (netdata_systemd_service_memory_usage_MiB_average{instance=~\"$instance\", service_name=~\"$service\", dimension=\"swap\"})", + "expr": "sum by(dimension, service_name) (netdata_systemd_service_memory_usage_MiB_average{hostname=~\"$hostname\", service_name=~\"$service\", dimension=\"swap\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -487,6 +492,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -597,6 +603,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -604,7 +611,7 @@ "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", - "expr": "netdata_system_cpu_some_pressure_stall_time_ms_average{instance=~\"$instance\"} * -1", + "expr": "netdata_system_cpu_some_pressure_stall_time_ms_average{hostname=~\"$hostname\"} * -1", "hide": false, "instant": false, "legendFormat": "some stall time", @@ -618,7 +625,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(dimension, service_name) (netdata_systemd_service_cpu_utilization_percentage_average{instance=~\"$instance\", service_name=~\"$service\"})", + "expr": "sum by(dimension, service_name) (netdata_systemd_service_cpu_utilization_percentage_average{hostname=~\"$hostname\", service_name=~\"$service\"})", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -649,6 +656,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -814,6 +822,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -821,7 +830,7 @@ "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", - "expr": "netdata_system_io_full_pressure_stall_time_ms_average{instance=~\"$instance\"} * -1", + "expr": "netdata_system_io_full_pressure_stall_time_ms_average{hostname=~\"$hostname\"} * -1", "hide": false, "instant": false, "legendFormat": "full stall time", @@ -878,6 +887,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -943,6 +953,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -950,7 +961,7 @@ "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "editorMode": "code", - "expr": "netdata_disk_io_KiB_persec_average{instance=~\"$instance\", chart=~\"disk.sd.+\"}", + "expr": "netdata_disk_io_KiB_persec_average{hostname=~\"$hostname\", chart=~\"disk.sd.+\"}", "instant": false, "legendFormat": "{{device}} / {{dimension}}", "range": true, @@ -1031,7 +1042,8 @@ "layout": "auto" }, "tooltip": { - "show": true, + "mode": "single", + "showColorScale": false, "yHistogram": false }, "yAxis": { @@ -1041,7 +1053,7 @@ "unit": "s" } }, - "pluginVersion": "10.2.0", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1049,7 +1061,7 @@ "uid": "cd6cc53e-840c-484d-85f7-96fede324006" }, "editorMode": "code", - "expr": "{unit=\"nginx.service\"} | pattern \"<_> <_> \" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | request_time > 100", + "expr": "{hostname=~\"$hostname\",unit=\"nginx.service\"} | pattern \"<_> <_> \" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | request_time > 100", "legendFormat": "{{server_name}}", "queryType": "range", "refId": "A" @@ -1145,6 +1157,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", @@ -1175,7 +1188,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1210,7 +1224,7 @@ "sort": "none" } }, - "pluginVersion": "10.2.0", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1218,7 +1232,7 @@ "uid": "cd6cc53e-840c-484d-85f7-96fede324006" }, "editorMode": "code", - "expr": "{unit=\"nginx.service\"} | pattern \"<_> <_> \" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | request_time > 100", + "expr": "{hostname=~\"$hostname\",unit=\"nginx.service\"} | pattern \"<_> <_> \" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | request_time > 100", "legendFormat": "", "queryType": "range", "refId": "A" @@ -1317,7 +1331,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1354,7 +1369,7 @@ }, "showHeader": true }, - "pluginVersion": "10.2.0", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1362,7 +1377,7 @@ "uid": "cd6cc53e-840c-484d-85f7-96fede324006" }, "editorMode": "code", - "expr": "{unit=\"nginx.service\"} | pattern \"<_> <_> \" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | request_time > 1", + "expr": "{hostname=~\"$hostname\",unit=\"nginx.service\"} | pattern \"<_> <_> \" | line_format \"{{.line}}\" | json | __error__ != \"JSONParserErr\" | request_time > 1", "queryType": "range", "refId": "A" } @@ -1411,7 +1426,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1472,7 +1488,7 @@ }, "showHeader": true }, - "pluginVersion": "10.2.0", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -1480,7 +1496,7 @@ "uid": "cd6cc53e-840c-484d-85f7-96fede324006" }, "editorMode": "code", - "expr": "{unit=\"postgresql.service\"} | regexp \".*duration: (?P[0-9.]+) ms (?P.*)\" | duration_ms > 500 | __error__ != \"LabelFilterErr\"", + "expr": "{hostname=~\"$hostname\",unit=\"postgresql.service\"} | regexp \".*duration: (?P[0-9.]+) ms (?P.*)\" | duration_ms > 500 | __error__ != \"LabelFilterErr\"", "queryType": "range", "refId": "A" } @@ -1513,56 +1529,55 @@ "type": "table" } ], + "preload": false, "refresh": "1m", - "schemaVersion": 38, + "schemaVersion": 40, "tags": [], "templating": { "list": [ { "allValue": ".*", "current": { - "selected": true, "text": [ - "127.0.0.1:19999" + "baryum" ], "value": [ - "127.0.0.1:19999" + "baryum" ] }, "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, - "definition": "label_values(netdata_systemd_service_unit_state_state_average,instance)", - "hide": 0, + "definition": "label_values(netdata_systemd_service_unit_state_state_average,hostname)", "includeAll": false, "multi": true, - "name": "instance", + "name": "hostname", "options": [], "query": { "qryType": 1, - "query": "label_values(netdata_systemd_service_unit_state_state_average,instance)", + "query": "label_values(netdata_systemd_service_unit_state_state_average,hostname)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" }, { "allValue": ".*", "current": { - "selected": false, - "text": "All", - "value": "$__all" + "text": [ + "All" + ], + "value": [ + "$__all" + ] }, "datasource": { "type": "prometheus", "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" }, "definition": "label_values(netdata_systemd_service_unit_state_state_average,unit_name)", - "hide": 0, "includeAll": true, "multi": true, "name": "service", @@ -1574,8 +1589,6 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" } ] @@ -1588,6 +1601,6 @@ "timezone": "", "title": "Performance", "uid": "e01156bf-cdba-42eb-9845-a401dd634d41", - "version": 54, + "version": 82, "weekStart": "" } diff --git a/modules/blocks/monitoring/dashboards/SSL.json b/modules/blocks/monitoring/dashboards/SSL.json new file mode 100644 index 00000000..9c82733a --- /dev/null +++ b/modules/blocks/monitoring/dashboards/SSL.json @@ -0,0 +1,143 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 16, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "transparent", + "value": 604808 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "ssl_certificate_expiry_seconds", + "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Certificate Remaining Validity", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "SSL Certificates", + "uid": "ae818js0bvw8wb", + "version": 8, + "weekStart": "" +} diff --git a/modules/blocks/monitoring/dashboards/Scraping Jobs.json b/modules/blocks/monitoring/dashboards/Scraping Jobs.json new file mode 100644 index 00000000..24df4879 --- /dev/null +++ b/modules/blocks/monitoring/dashboards/Scraping Jobs.json @@ -0,0 +1,286 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "rate(net_conntrack_dialer_conn_failed_total{hostname=~\"$hostname\"}[2m]) > 0", + "instant": false, + "legendFormat": "{{dialer_name}} - {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "alignValue": "center", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "prometheus_sd_discovered_targets{hostname=~\"$hostname\"}", + "hide": false, + "instant": false, + "legendFormat": "{{config}}", + "range": true, + "refId": "All" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "label_replace(increase((sum by(dialer_name) (net_conntrack_dialer_conn_failed_total{hostname=~\"$hostname\"}))[15m:1m]), \"config\", \"$1\", \"dialer_name\", \"(.*)\") > 10", + "hide": false, + "instant": false, + "legendFormat": "{{dialer_name}}", + "range": true, + "refId": "Failed" + } + ], + "title": "Scraping jobs", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "keepLabels": [ + "config" + ], + "mode": "columns" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "prometheus_sd_discovered_targets": true + }, + "indexByName": {}, + "renameByName": { + "prometheus_sd_discovered_targets": "" + } + } + }, + { + "id": "partitionByValues", + "options": { + "fields": [ + "config" + ] + } + } + ], + "type": "state-timeline" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "baryum", + "value": "baryum" + }, + "definition": "label_values(up,hostname)", + "includeAll": false, + "name": "hostname", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(up,hostname)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Scraping Jobs", + "uid": "debb763d-77aa-47bd-9290-2e02583c8ed2", + "version": 15, + "weekStart": "" +} diff --git a/modules/blocks/monitoring/dashboards/Torrents.json b/modules/blocks/monitoring/dashboards/Torrents.json new file mode 100644 index 00000000..563225fa --- /dev/null +++ b/modules/blocks/monitoring/dashboards/Torrents.json @@ -0,0 +1,1155 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 12, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 15, + "panels": [], + "title": "Torrent", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 19, + "maxPerRow": 3, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.4.0", + "repeat": "mountpoint", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "node_filesystem_size_bytes{hostname=~\"$hostname\",mountpoint=\"$mountpoint\"} - node_filesystem_free_bytes{hostname=~\"$hostname\",mountpoint=\"$mountpoint\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "node_filesystem_size_bytes{hostname=~\"$hostname\",mountpoint=\"$mountpoint\"}", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + }, + { + "conditions": [ + { + "evaluator": { + "params": [ + 0, + 0 + ], + "type": "gt" + }, + "query": { + "params": [] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "name": "Expression", + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "$B*0.95", + "hide": false, + "refId": "D", + "type": "math" + } + ], + "title": "$mountpoint Used Space", + "transformations": [ + { + "id": "configFromData", + "options": { + "applyTo": { + "id": "byFrameRefID", + "options": "A" + }, + "configRefId": "B", + "mappings": [ + { + "fieldName": "Time", + "handlerKey": "__ignore" + }, + { + "fieldName": "device", + "handlerKey": "__ignore" + }, + { + "fieldName": "domain", + "handlerKey": "__ignore" + }, + { + "fieldName": "fstype", + "handlerKey": "__ignore" + }, + { + "fieldName": "hostname", + "handlerKey": "__ignore" + }, + { + "fieldName": "instance", + "handlerKey": "__ignore" + }, + { + "fieldName": "job", + "handlerKey": "__ignore" + }, + { + "fieldName": "mountpoint", + "handlerKey": "__ignore" + }, + { + "fieldName": "{__name__=\"node_filesystem_size_bytes\", device=\"data/movies\", fstype=\"zfs\", instance=\"127.0.0.1:9112\", job=\"node\", mountpoint=\"/srv/movies\"}", + "handlerKey": "max" + }, + { + "fieldName": "node_filesystem_size_bytes {__name__=\"node_filesystem_size_bytes\", device=\"data/movies\", fstype=\"zfs\", instance=\"127.0.0.1:9112\", job=\"node\", mountpoint=\"/srv/movies\"}", + "handlerKey": "max" + } + ] + } + }, + { + "id": "configFromData", + "options": { + "applyTo": { + "id": "byFrameRefID", + "options": "A" + }, + "configRefId": "D", + "mappings": [ + { + "fieldName": "D {__name__=\"node_filesystem_size_bytes\", device=\"data/movies\", fstype=\"zfs\", instance=\"127.0.0.1:9112\", job=\"node\", mountpoint=\"/srv/movies\"}", + "handlerArguments": { + "threshold": { + "color": "red" + } + }, + "handlerKey": "threshold1" + } + ] + } + } + ], + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 4, + "x": 3, + "y": 1 + }, + "id": 17, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "exemplar": true, + "expr": "deluge_torrents{hostname=~\"$hostname\"}", + "instant": false, + "interval": "", + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "title": "Torrent States", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 17, + "x": 7, + "y": 1 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "deluge_torrent_done_total{hostname=~\"$hostname\",state=\"downloading\",name=~\"$torrent\"} / deluge_torrent_size_total{hostname=~\"$hostname\",state=\"downloading\",name=~\"$torrent\"}", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "title": "In Progress Downloads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "-1": { + "index": 0, + "text": "Never" + } + }, + "type": "value" + } + ], + "max": 86400, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-red", + "value": null + }, + { + "color": "semi-dark-green", + "value": 0 + }, + { + "color": "semi-dark-orange", + "value": 86400 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 7, + "y": 8 + }, + "id": 31, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "exemplar": false, + "expr": "deluge_torrent_time_since_download{hostname=~\"$hostname\",state=\"downloading\",name=~\"$torrent\"}", + "instant": true, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Last Download", + "transformations": [ + { + "id": "seriesToRows", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "Value" + } + ] + } + }, + { + "id": "rowsToFields", + "options": { + "mappings": [ + { + "fieldName": "Time", + "handlerKey": "__ignore" + }, + { + "fieldName": "Value", + "handlerKey": "field.value" + }, + { + "fieldName": "Metric", + "handlerKey": "field.name" + } + ] + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "1642487291": { + "color": "semi-dark-red", + "index": 0, + "text": "Never" + } + }, + "type": "value" + } + ], + "max": 3600, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-green", + "value": null + }, + { + "color": "#EAB839", + "value": 86400 + }, + { + "color": "semi-dark-red", + "value": 1642487290 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 13, + "y": 8 + }, + "id": 29, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "exemplar": false, + "expr": "time()-deluge_torrent_last_seen_complete{hostname=~\"$hostname\",state=\"downloading\",name=~\"$torrent\"}", + "instant": true, + "interval": "", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "Last Seen Completed", + "transformations": [ + { + "id": "seriesToRows", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Value" + } + ] + } + }, + { + "id": "rowsToFields", + "options": { + "mappings": [ + { + "fieldName": "Time", + "handlerKey": "__ignore" + }, + { + "fieldName": "Value", + "handlerKey": "field.value" + }, + { + "fieldName": "Metric", + "handlerKey": "field.name" + } + ] + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 19, + "y": 8 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(device) (rate(node_network_receive_bytes_total{hostname=~\"$hostname\",device=~\"tun.*\"}[5m]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "in: {{ device }}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "exemplar": true, + "expr": "-avg by(device) (rate(node_network_transmit_bytes_total{device=~\"tun.*\"}[5m]))", + "hide": false, + "interval": "", + "legendFormat": "out: {{ device }}", + "range": true, + "refId": "B" + } + ], + "title": "VPN Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 9, + "panels": [], + "title": "Services", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "exemplar": true, + "expr": "netdata_systemd_service_unit_state_state_average{hostname=~\"$hostname\",unit_name=~\"deluged|delugeweb|openvpn.+\",dimension=\"active\"}", + "interval": "", + "legendFormat": "{{unit_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Services Up", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 2, + "options": { + "dedupStrategy": "exact", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=\"deluged.service\",level=~\"$level\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Deluge Logs", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 21 + }, + "id": 4, + "options": { + "dedupStrategy": "exact", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=\"deluged.service\"} |= \"on_alert_external_ip\" | regexp \".+on_alert_external_ip: (?P.+)\" | line_format \"{{.ip}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Latest External IPs", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 21 + }, + "id": 13, + "options": { + "dedupStrategy": "exact", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=~\"openvpn.+.service\"} |= \"config -s listen_interface\" | pattern \"<_> listen_interface '\" | line_format \"{{.ip}}\"", + "queryType": "range", + "refId": "A" + } + ], + "title": "Latest Interface IPs", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 16, + "x": 8, + "y": 26 + }, + "id": 7, + "options": { + "dedupStrategy": "exact", + "enableLogDetails": false, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cd6cc53e-840c-484d-85f7-96fede324006" + }, + "editorMode": "code", + "expr": "{hostname=~\"$hostname\",unit=~\"openvpn.+.service\",level=~\"$level\"}", + "legendFormat": "", + "queryType": "range", + "refId": "A" + } + ], + "title": "VPN Logs", + "type": "logs" + } + ], + "preload": false, + "refresh": "10s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "$__all" + ], + "value": [ + "$__all" + ] + }, + "hide": 2, + "includeAll": true, + "multi": true, + "name": "mountpoint", + "options": [ + { + "selected": false, + "text": "/srv/movies", + "value": "/srv/movies" + }, + { + "selected": false, + "text": "/srv/music", + "value": "/srv/music" + }, + { + "selected": false, + "text": "/srv/series", + "value": "/srv/series" + } + ], + "query": "/srv/movies,/srv/music,/srv/series", + "type": "custom" + }, + { + "current": { + "text": "baryum", + "value": "baryum" + }, + "definition": "label_values(up,hostname)", + "name": "hostname", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(up,hostname)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "definition": "deluge_torrent_done_total", + "includeAll": true, + "multi": true, + "name": "torrent", + "options": [], + "query": { + "qryType": 4, + "query": "deluge_torrent_done_total", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "/.*name=\"(?[^\"]+)\".*/", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Torrents", + "uid": "Bg5L6T17k", + "version": 22, + "weekStart": "" +} diff --git a/modules/blocks/monitoring/docs/assets/dashboards_Deluge_1.png b/modules/blocks/monitoring/docs/assets/dashboards_Deluge_1.png new file mode 100644 index 00000000..d8d36309 Binary files /dev/null and b/modules/blocks/monitoring/docs/assets/dashboards_Deluge_1.png differ diff --git a/modules/blocks/monitoring/docs/assets/dashboards_Deluge_2.png b/modules/blocks/monitoring/docs/assets/dashboards_Deluge_2.png new file mode 100644 index 00000000..71658c3d Binary files /dev/null and b/modules/blocks/monitoring/docs/assets/dashboards_Deluge_2.png differ diff --git a/modules/blocks/monitoring/docs/default.md b/modules/blocks/monitoring/docs/default.md index a0774efb..c78462f4 100644 --- a/modules/blocks/monitoring/docs/default.md +++ b/modules/blocks/monitoring/docs/default.md @@ -101,8 +101,8 @@ This dashboard is meant to be the first stop to understand why a service is misb ![](./assets/dashboards_Errors_1.png) ![](./assets/dashboards_Errors_2.png) -The yellow and red dashed vertical bars correspond to the [Requests Error Budget -Alert](#blocks-monitoring-budget-alerts) firing. +The yellow and red dashed vertical bars correspond to the +[Requests Error Budget Alert](#blocks-monitoring-budget-alerts) firing. ## Performance Dashboard {#blocks-monitoring-performance-dashboard} @@ -112,6 +112,17 @@ This dashboard is meant to be the first stop to understand why a service is perf ![Performance Dashboard Middle Part](./assets/dashboards_Performance_2.png) ![Performance Dashboard Bottom Part](./assets/dashboards_Performance_3.png) +## Nextcloud Dashboard {#blocks-monitoring-nextcloud-dashboard} + +See [Nextcloud service](./services-nextcloud.html#services-nextcloudserver-dashboard) manual. + +## Deluge Dashboard {#blocks-monitoring-deluge-dashboard} + +This dashboard is used to monitor a [deluge](./services-deluge.html) instance. + +![Deluge Dashboard Top Part](./assets/dashboards_Deluge_1.png) +![Deluge Dashboard Bottom Part](./assets/dashboards_Deluge_2.png) + ## Requests Error Budget Alert {#blocks-monitoring-budget-alerts} This alert will fire when the ratio between number of requests getting a 5XX response from a service @@ -119,3 +130,11 @@ and the total requests to that service exceeds 1%. ![Error Dashboard Top Part](./assets/alert_rules_5xx_1.png) ![Error Dashboard Bottom Part](./assets/alert_rules_5xx_2.png) + +## Options Reference {#blocks-monitoring-options} + +```{=include=} options +id-prefix: blocks-monitoring-options- +list-id: selfhostblocks-blocks-monitoring-options +source: @OPTIONS_JSON@ +``` diff --git a/modules/blocks/monitoring/rules.json b/modules/blocks/monitoring/rules.json index 192fbc60..f4482b8e 100644 --- a/modules/blocks/monitoring/rules.json +++ b/modules/blocks/monitoring/rules.json @@ -123,7 +123,134 @@ "summary": "The error budget for a service for the last 1 hour is under 99%" }, "labels": { - "": "", + "role": "sysadmin" + }, + "isPaused": false + }, + { + "uid": "ee817l3a88s1sd", + "title": "Certificate Did Not Renew", + "condition": "C", + "data": [ + { + "refId": "A", + "relativeTimeRange": { + "from": 1800, + "to": 0 + }, + "datasourceUid": "df80f9f5-97d7-4112-91d8-72f523a02b09", + "model": { + "adhocFilters": [], + "datasource": { + "type": "prometheus", + "uid": "df80f9f5-97d7-4112-91d8-72f523a02b09" + }, + "editorMode": "code", + "expr": "ssl_certificate_expiry_seconds", + "interval": "", + "intervalMs": 15000, + "legendFormat": "{{exported_hostname}}: {{subject}} {{path}}", + "maxDataPoints": 43200, + "range": true, + "refId": "A" + } + }, + { + "refId": "B", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "intervalMs": 1000, + "maxDataPoints": 43200, + "reducer": "last", + "refId": "B", + "type": "reduce" + } + }, + { + "refId": "C", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [ + 604800 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "B", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "C", + "type": "threshold" + } + } + ], + "dashboardUid": "ae818js0bvw8wb", + "panelId": 3, + "noDataState": "NoData", + "execErrState": "Error", + "for": "20m", + "annotations": { + "__dashboardUid__": "ae818js0bvw8wb", + "__panelId__": "3", + "description": "The expiry date of the certificate is 1 week from now.", + "summary": "Certificate did not renew on time." + }, + "labels": { "role": "sysadmin" }, "isPaused": false diff --git a/modules/blocks/postgresql.nix b/modules/blocks/postgresql.nix index 4bb1ea5c..7472255a 100644 --- a/modules/blocks/postgresql.nix +++ b/modules/blocks/postgresql.nix @@ -55,6 +55,7 @@ in Backup configuration. ''; + default = {}; type = lib.types.submodule { options = contracts.databasebackup.mkRequester { user = "postgres"; @@ -169,10 +170,5 @@ in (upgrade-script 15 16) ]; } - { - # Seems superfluous but otherwise we get: - # The option `shb.postgresql.databasebackup' was accessed but has no value defined. - shb.postgresql.databasebackup = {}; - } ]); } diff --git a/modules/blocks/ssl.nix b/modules/blocks/ssl.nix index 0ad0f848..0b77b829 100644 --- a/modules/blocks/ssl.nix +++ b/modules/blocks/ssl.nix @@ -4,6 +4,9 @@ let cfg = config.shb.certs; contracts = pkgs.callPackage ../contracts {}; + + inherit (builtins) dirOf; + inherit (lib) flatten mapAttrsToList unique; in { options.shb.certs = { @@ -362,7 +365,7 @@ in cat /etc/static/ssl/certs/ca-bundle.crt > /etc/ssl/certs/ca-bundle.crt cat /etc/static/ssl/certs/ca-bundle.crt > /etc/ssl/certs/ca-certificates.crt - for file in ${lib.concatStringsSep " " (lib.mapAttrsToList (_name: caCfg: caCfg.paths.cert) cfg.cas.selfsigned)}; do + for file in ${lib.concatStringsSep " " (mapAttrsToList (_name: caCfg: caCfg.paths.cert) cfg.cas.selfsigned)}; do cat "$file" >> /etc/ssl/certs/ca-bundle.crt cat "$file" >> /etc/ssl/certs/ca-certificates.crt done @@ -431,7 +434,7 @@ in } # Config for Let's Encrypt cert. { - users.users = lib.mkMerge (lib.mapAttrsToList (name: certCfg: { + users.users = lib.mkMerge (mapAttrsToList (name: certCfg: { ${certCfg.makeAvailableToUser}.extraGroups = lib.mkIf (!(isNull certCfg.makeAvailableToUser)) [ config.security.acme.defaults.group ]; @@ -447,7 +450,7 @@ in server = lib.mkIf certCfg.stagingServer "https://acme-staging-v02.api.letsencrypt.org/directory"; }; }) certCfg.extraDomains; - in lib.mkMerge (lib.flatten (lib.mapAttrsToList (name: certCfg: + in lib.mkMerge (flatten (mapAttrsToList (name: certCfg: [{ "${name}" = { extraDomainNames = [ certCfg.domain ] ++ certCfg.extraDomains; @@ -470,7 +473,7 @@ in enableACME = true; }; }) extraDomains; - in lib.mkMerge (lib.flatten (lib.mapAttrsToList (name: certCfg: + in lib.mkMerge (flatten (mapAttrsToList (name: certCfg: lib.optionals (certCfg.dnsProvider == null) ( [{ virtualHosts."${name}" = { @@ -482,7 +485,7 @@ in )) cfg.certs.letsencrypt)); systemd.services = let - extraDomainsCfg = certCfg: lib.flatten (map (name: + extraDomainsCfg = certCfg: flatten (map (name: lib.optionals (certCfg.additionalEnvironment != {} && certCfg.dnsProvider == null) [{ "acme-${name}".environment = certCfg.additionalEnvironment; }] @@ -493,7 +496,7 @@ in }; }] ) certCfg.extraDomains); - in lib.mkMerge (lib.flatten (lib.mapAttrsToList (name: certCfg: + in lib.mkMerge (flatten (mapAttrsToList (name: certCfg: lib.optionals (certCfg.additionalEnvironment != {} && certCfg.dnsProvider == null) [{ "acme-${certCfg.domain}".environment = certCfg.additionalEnvironment; }] @@ -505,6 +508,37 @@ in }] ++ lib.optionals (certCfg.dnsProvider == null) (extraDomainsCfg certCfg) ) cfg.certs.letsencrypt)); + + services.prometheus.exporters.node-cert = { + enable = true; + listenAddress = "127.0.0.1"; + user = "acme"; + paths = let + pathCfg = name: certCfg: + let + mainDomainPaths = map dirOf [ certCfg.paths.cert certCfg.paths.key ]; + # Not sure this will work for all cases. + mainPath = dirOf (dirOf certCfg.paths.cert); + extraDomainsPath = map (x: "${mainPath}/${x}") certCfg.extraDomains; + in + mainDomainPaths ++ extraDomainsPath; + in + unique (flatten (mapAttrsToList pathCfg cfg.certs.letsencrypt)); + }; + + services.prometheus.scrapeConfigs = let + scrapeCfg = name: certCfg: [{ + job_name = "node-cert-${name}"; + static_configs = [{ + targets = ["127.0.0.1:${toString config.services.prometheus.exporters.node-cert.port}"]; + labels = { + "hostname" = config.networking.hostName; + "domain" = certCfg.domain; + }; + }]; + }]; + in + flatten (mapAttrsToList scrapeCfg cfg.certs.letsencrypt); } ]; } diff --git a/modules/services/arr.nix b/modules/services/arr.nix index f5c58ea7..b08b793f 100644 --- a/modules/services/arr.nix +++ b/modules/services/arr.nix @@ -267,6 +267,7 @@ let policy = "bypass"; resources = extraBypassResources ++ [ "^/api.*" + "^/feed.*" ]; } { @@ -319,6 +320,7 @@ let description = '' Backup configuration. ''; + default = {}; type = lib.types.submodule { options = contracts.backup.mkRequester { user = name; diff --git a/modules/services/deluge.nix b/modules/services/deluge.nix index 11256b11..c5bf6cbb 100644 --- a/modules/services/deluge.nix +++ b/modules/services/deluge.nix @@ -247,6 +247,7 @@ in description = '' Backup configuration. ''; + default = {}; type = lib.types.submodule { options = contracts.backup.mkRequester { user = "deluge"; @@ -261,7 +262,7 @@ in type = lib.types.nullOr (lib.types.enum ["critical" "error" "warning" "info" "debug"]); description = "Enable logging."; default = null; - example = true; + example = "info"; }; }; @@ -391,6 +392,10 @@ in job_name = "deluge"; static_configs = [{ targets = ["127.0.0.1:${toString config.services.prometheus.exporters.deluge.port}"]; + labels = { + "hostname" = config.networking.hostName; + "domain" = cfg.domain; + }; }]; } ]; diff --git a/modules/services/forgejo.nix b/modules/services/forgejo.nix index 3d5b1cbb..73991320 100644 --- a/modules/services/forgejo.nix +++ b/modules/services/forgejo.nix @@ -244,6 +244,7 @@ in description = '' Backup configuration. ''; + default = {}; type = lib.types.submodule { options = contracts.backup.mkRequester { user = options.services.forgejo.user.value; diff --git a/modules/services/hledger.nix b/modules/services/hledger.nix index 99bbd03f..efdd8018 100644 --- a/modules/services/hledger.nix +++ b/modules/services/hledger.nix @@ -49,15 +49,17 @@ in }; authEndpoint = lib.mkOption { - type = lib.types.str; + type = lib.types.nullOr lib.types.str; description = "OIDC endpoint for SSO"; example = "https://authelia.example.com"; + default = null; }; backup = lib.mkOption { description = '' Backup configuration. ''; + default = {}; type = lib.types.submodule { options = contracts.backup.mkRequester { user = "hledger"; diff --git a/modules/services/home-assistant.nix b/modules/services/home-assistant.nix index 8f2189f5..834a1586 100644 --- a/modules/services/home-assistant.nix +++ b/modules/services/home-assistant.nix @@ -139,10 +139,47 @@ in }; }; + voice = lib.mkOption { + description = "Options related to voice service."; + default = {}; + type = lib.types.submodule { + options = { + speech-to-text = lib.mkOption { + description = '' + Wyoming piper servers. + + https://search.nixos.org/options?channel=23.11&from=0&size=50&sort=relevance&type=packages&query=services.wyoming.piper.servers + ''; + type = lib.types.attrsOf lib.types.anything; + default = {}; + }; + text-to-speech = lib.mkOption { + description = '' + Wyoming faster-whisper servers. + + https://search.nixos.org/options?channel=23.11&from=0&size=50&sort=relevance&type=packages&query=services.wyoming.faster-whisper.servers + ''; + type = lib.types.attrsOf lib.types.anything; + default = {}; + }; + wakeword = lib.mkOption { + description = '' + Wyoming open wakework servers. + + https://search.nixos.org/options?channel=23.11&from=0&size=50&sort=relevance&type=packages&query=services.wyoming.openwakeword + ''; + type = lib.types.anything; + default = { enable = false; }; + }; + }; + }; + }; + backup = lib.mkOption { description = '' Backup configuration. ''; + default = {}; type = lib.types.submodule { options = contracts.backup.mkRequester { user = "hass"; @@ -266,6 +303,10 @@ in }; }; + services.wyoming.piper.servers = cfg.voice.text-to-speech; + services.wyoming.faster-whisper.servers = cfg.voice.speech-to-text; + services.wyoming.openwakeword = cfg.voice.wakeword; + services.nginx.virtualHosts."${fqdn}" = { http2 = true; @@ -314,6 +355,7 @@ in "f ${config.services.home-assistant.configDir}/automations.yaml 0755 hass hass" "f ${config.services.home-assistant.configDir}/scenes.yaml 0755 hass hass" "f ${config.services.home-assistant.configDir}/scripts.yaml 0755 hass hass" + "d /var/lib/hass/backups 0750 hass hass" ]; }; } diff --git a/modules/services/jellyfin.nix b/modules/services/jellyfin.nix index adaeef81..804afac5 100644 --- a/modules/services/jellyfin.nix +++ b/modules/services/jellyfin.nix @@ -154,6 +154,7 @@ in description = '' Backup configuration. ''; + default = {}; type = lib.types.submodule { options = contracts.backup.mkRequester { user = "jellyfin"; @@ -289,6 +290,10 @@ in static_configs = [ { targets = ["127.0.0.1:8096"]; + labels = { + "hostname" = config.networking.hostName; + "domain" = cfg.domain; + }; } ]; }]; diff --git a/modules/services/nextcloud-server.nix b/modules/services/nextcloud-server.nix index c0fed189..a78d175e 100644 --- a/modules/services/nextcloud-server.nix +++ b/modules/services/nextcloud-server.nix @@ -168,7 +168,11 @@ in phpFpmPoolSettings = lib.mkOption { type = lib.types.nullOr (lib.types.attrsOf lib.types.anything); description = "Settings for PHPFPM."; - default = null; + default = { + "pm" = "static"; + "pm.max_children" = 5; + "pm.start_servers" = 5; + }; example = lib.literalExpression '' { "pm" = "dynamic"; @@ -183,6 +187,27 @@ in ''; }; + phpFpmPrometheusExporter = lib.mkOption { + description = "Settings for exporting"; + default = {}; + + type = lib.types.submodule { + options = { + enable = lib.mkOption { + description = "Enable export of php-fpm metrics to Prometheus."; + type = lib.types.bool; + default = true; + }; + + port = lib.mkOption { + description = "Port on which the exporter will listen."; + type = lib.types.port; + default = 8300; + }; + }; + }; + }; + apps = lib.mkOption { description = '' Applications to enable in Nextcloud. Enabling an application here will also configure @@ -522,6 +547,7 @@ in description = '' Backup configuration. ''; + default = {}; type = lib.types.submodule { options = contracts.backup.mkRequester { user = "nextcloud"; @@ -738,11 +764,53 @@ in ]; systemd.timers.nextcloud-cron.requires = cfg.mountPointServices; systemd.timers.nextcloud-cron.after = cfg.mountPointServices; + # This is needed to be able to run the cron job before opening the app for the first time. + # Otherwise the cron job fails while searching for this directory. + systemd.services.nextcloud-setup.script = '' + mkdir -p ${cfg.dataDir}/data/appdata_$(${occ} config:system:get instanceid)/theming/global + ''; systemd.services.nextcloud-setup.requires = cfg.mountPointServices; systemd.services.nextcloud-setup.after = cfg.mountPointServices; }) + (lib.mkIf cfg.phpFpmPrometheusExporter.enable { + services.prometheus.exporters.php-fpm = { + enable = true; + user = "nginx"; + port = cfg.phpFpmPrometheusExporter.port; + listenAddress = "127.0.0.1"; + extraFlags = [ + "--phpfpm.scrape-uri=tcp://127.0.0.1:${toString (cfg.phpFpmPrometheusExporter.port -1)}/status?full" + ]; + }; + + services.nextcloud = { + poolSettings = { + "pm.status_path" = "/status"; + # Need to use TCP connection to get status. + # I couldn't get PHP-FPM exporter to work with a unix socket. + # + # I also tried to server the status page at /status.php + # but fcgi doesn't like the returned headers. + "pm.status_listen" = "127.0.0.1:${toString (cfg.phpFpmPrometheusExporter.port -1)}"; + }; + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "phpfpm-nextcloud"; + static_configs = [{ + targets = ["127.0.0.1:${toString cfg.phpFpmPrometheusExporter.port}"]; + labels = { + "hostname" = config.networking.hostName; + "domain" = cfg.domain; + }; + }]; + } + ]; + }) + (lib.mkIf (cfg.enable && cfg.apps.onlyoffice.enable) { assertions = [ { diff --git a/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_1.png b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_1.png new file mode 100644 index 00000000..3e3b3c16 Binary files /dev/null and b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_1.png differ diff --git a/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_2.png b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_2.png new file mode 100644 index 00000000..f7c3548b Binary files /dev/null and b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_2.png differ diff --git a/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_3.png b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_3.png new file mode 100644 index 00000000..698e67a4 Binary files /dev/null and b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_3.png differ diff --git a/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_4.png b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_4.png new file mode 100644 index 00000000..565273e2 Binary files /dev/null and b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_4.png differ diff --git a/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_5.png b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_5.png new file mode 100644 index 00000000..01de3f0d Binary files /dev/null and b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_5.png differ diff --git a/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_6.png b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_6.png new file mode 100644 index 00000000..debb0c26 Binary files /dev/null and b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_6.png differ diff --git a/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_7.png b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_7.png new file mode 100644 index 00000000..6c0a48f2 Binary files /dev/null and b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_7.png differ diff --git a/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_error_parsing.png b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_error_parsing.png new file mode 100644 index 00000000..2610e128 Binary files /dev/null and b/modules/services/nextcloud-server/docs/assets/dashboards_Nextcloud_error_parsing.png differ diff --git a/modules/services/nextcloud-server/docs/default.md b/modules/services/nextcloud-server/docs/default.md index 1e22c3e0..04b20d1c 100644 --- a/modules/services/nextcloud-server/docs/default.md +++ b/modules/services/nextcloud-server/docs/default.md @@ -29,7 +29,7 @@ It is based on the nixpkgs Nextcloud server and provides opinionated defaults. - Forces PostgreSQL as the database. - Forces Redis as the cache and sets good defaults. - Backup of the [`shb.nextcloud.dataDir`][dataDir] through the [backup block](./blocks-backup.html). -- Monitoring of reverse proxy, PHP-FPM, and database backups through the [monitoring +- [Dashboard](#services-nextcloudserver-dashboard) for monitoring of reverse proxy, PHP-FPM, and database backups through the [monitoring block](./blocks-monitoring.html). - [Integration Tests](@REPO@/test/services/nextcloud.nix) - Tests system cron job is setup correctly. @@ -311,6 +311,9 @@ I don't have a good heuristic for what are good values here but what I found is that you don't want too high of a `max_children` value to avoid I/O strain on the hard drives, especially if you use spinning drives. +To see the effect of your settings, +go to the provided [Grafana dashboard](#services-nextcloudserver-dashboard). + ### Tweak PostgreSQL Settings {#services-nextcloudserver-usage-postgres} These settings will impact all databases since the NixOS Postgres module @@ -353,6 +356,9 @@ shb.nextcloud.postgresSettings = { }; ``` +To see the effect of your settings, +go to the provided [Grafana dashboard](#services-nextcloudserver-dashboard). + ### Backup {#services-nextcloudserver-usage-backup} Backing up Nextcloud data files using the [Restic block](blocks-restic.html) is done like so: @@ -526,12 +532,80 @@ by issuing the command `nextcloud-occ config:system:delete instanceid`. Head over to the [Nextcloud demo](demo-nextcloud-server.html) for a demo that installs Nextcloud with or without LDAP integration on a VM with minimal manual steps. -## Maintenance {#services-nextcloudserver-maintenance} - -On the command line, the `occ` tool is called `nextcloud-occ`. +## Dashboard {#services-nextcloudserver-dashboard} + +The dashboard is added to Grafana automatically under "Self Host Blocks > Nextcloud" +as long as the Nextcloud service is [enabled][] +as well as the [monitoring block][]. + +[enabled]: #services-nextcloudserver-options-shb.nextcloud.enable +[monitoring block]: ./blocks-monitoring.html#blocks-monitoring-options-shb.monitoring.enable + +- The *General* section shows Nextcloud related services. + This includes cronjobs, Redis and backup jobs. +- *CPU* shows stall time which means CPU is maxed out. + This graph is inverted so having a small area at the top means the stall time is low. +- *Memory* shows stall time which means some job is waiting on memory to be allocated. + This graph is inverted so having a small area at the top means the stall time is low. + Some stall time will always be present. Under 10% is fine + but having constantly over 50% usually means available memory is low and SWAP is being used. + *Memory* also shows available memory which is the remaining allocatable memory. +- Caveat: *Network I/O* shows the network input and output for + all services running, not only those related to Nextcloud. +- *Disk I/O* shows "some" stall time which means some jobs were waiting on disk I/O. + Disk is usually the slowest bottleneck so having "some" stall time is not surprising. + Fixing this can be done by using disks allowing higher speeds or switching to SSDs. + If the "full" stall time is shown, this means _all_ jobs were waiting on disk i/o which + can be more worrying. This could indicate a failing disk if "full" stall time appeared recently. + These graphs are inverted so having a small area at the top means the stall time is low. + *Memory* also shows available memory which is the remaining allocatable memory. +![Nextcloud Dashboard First Part](./assets/dashboards_Nextcloud_1.png) +- *PHP-FPM Processes* shows how many processes are used by PHP-FPM. + The orange area goes from 80% to 90% of the maximum allowed processes. + The read area goes from 90% to 100% of the maximum allowed processes. + If the number of active processes reaches those areas once in a while, that's fine + but if it happens most of the time, the maximum allowed processes should be increased. +- *PHP-FPM Request Duration* shows one dot per request and how long it took. + Request time is fine if it is under 400ms. + If most requests take longer than that, some [tracing](#services-nextcloudserver-server-usage-tracing) + is required to understand which subsystem is taking some time. + That being said, maybe another graph in this dashboard will show + why the requests are slow - like disk + or other processes hoarding some resources running at the same time. +- *PHP-FPM Requests Queue Length* shows how many requests are waiting + to be picked up by a PHP-FPM process. Usually, this graph won't show + anything as long as the *PHP-FPM Processes* graph is not in the red area. + Fixing this requires also increasing the maximum allowed processes. +![Nextcloud Dashboard Second Part](./assets/dashboards_Nextcloud_2.png) +- *Requests Details* shows all requests to the Nextcloud service and the related headers. +- *5XX Requests Details* shows only the requests having a 500 to 599 http status. + Having any requests appearing here should be investigated as soon as possible. +![Nextcloud Dashboard Third Part](./assets/dashboards_Nextcloud_3.png) +- *Log: \* shows all logs from related systemd `.service` job. + Having no line here most often means the job ran + at a time not currently included in the time range of the dashboard. +![Nextcloud Dashboard Fourth Part](./assets/dashboards_Nextcloud_4.png) +![Nextcloud Dashboard Fifth Part](./assets/dashboards_Nextcloud_5.png) +- A lot of care has been taken to parse error messages correctly. + Nextcloud mixes json and non-json messages so extracting errors + from json messages was not that easy. + Also, the stacktrace is reduced. + The result though is IMO pretty nice as can be seen by the following screenshot. + The top line is the original json message and the bottom one is the parsed error. +![Nextcloud Dashboard Error Parsing](./assets/dashboards_Nextcloud_error_parsing.png) +- *Backup logs* show the output of the backup jobs. + Here, there are two backup jobs, one for the core files of Nextcloud + stored on an SSD which includes the appdata folder. + The other backup job is for the external data stored on HDDs which contain all user files. +![Nextcloud Dashboard Sixth Part](./assets/dashboards_Nextcloud_6.png) +- *Slow PostgreSQL queries* shows all database queries taking longer than 1s to run. +- *Redis* shows all Redis log output. +![Nextcloud Dashboard Seventh Part](./assets/dashboards_Nextcloud_7.png) ## Debug {#services-nextcloudserver-debug} +On the command line, the `occ` tool is called `nextcloud-occ`. + In case of an issue, check the logs for any systemd service mentioned in this section. On startup, the oneshot systemd service `nextcloud-setup.service` starts. After it finishes, the diff --git a/modules/services/vaultwarden.nix b/modules/services/vaultwarden.nix index 09415abb..83cf52a9 100644 --- a/modules/services/vaultwarden.nix +++ b/modules/services/vaultwarden.nix @@ -131,6 +131,7 @@ in description = '' Backup configuration. ''; + default = {}; type = lib.types.submodule { options = contracts.backup.mkRequester { user = "vaultwarden"; @@ -231,9 +232,5 @@ in # TODO: make this work. # It does not work because it leads to infinite recursion. # ${cfg.mount}.path = dataFolder; - - # Seems superfluous but otherwise we get: - # The option `shb.vaultwarden.backup' was accessed but has no value defined. - shb.vaultwarden.backup = {}; }; } diff --git a/patches/prometheusnodecertexporter.nix b/patches/prometheusnodecertexporter.nix new file mode 100644 index 00000000..03e20046 --- /dev/null +++ b/patches/prometheusnodecertexporter.nix @@ -0,0 +1,219 @@ +index f805920c5b87a..b67f41c4fb12c 100644 +--- a/nixos/modules/services/monitoring/prometheus/exporters.nix ++++ b/nixos/modules/services/monitoring/prometheus/exporters.nix +@@ -66,6 +66,7 @@ let + "nginx" + "nginxlog" + "node" ++ "node-cert" + "nut" + "nvidia-gpu" + "pgbouncer" +diff --git a/nixos/modules/services/monitoring/prometheus/exporters/node-cert.nix b/nixos/modules/services/monitoring/prometheus/exporters/node-cert.nix +new file mode 100644 +index 0000000000000..d8b2004e8e857 +--- /dev/null ++++ b/nixos/modules/services/monitoring/prometheus/exporters/node-cert.nix +@@ -0,0 +1,70 @@ ++{ ++ config, ++ lib, ++ pkgs, ++ ... ++}: ++ ++let ++ cfg = config.services.prometheus.exporters.node-cert; ++ inherit (lib) mkOption types concatStringsSep; ++in ++{ ++ port = 9141; ++ ++ extraOpts = { ++ paths = mkOption { ++ type = types.listOf types.str; ++ description = '' ++ List of paths to search for SSL certificates. ++ ''; ++ }; ++ ++ excludePaths = mkOption { ++ type = types.listOf types.str; ++ description = '' ++ List of paths to exclute from searching for SSL certificates. ++ ''; ++ default = [ ]; ++ }; ++ ++ includeGlobs = mkOption { ++ type = types.listOf types.str; ++ description = '' ++ List files matching a pattern to include. Uses Go blob pattern. ++ ''; ++ default = [ ]; ++ }; ++ ++ excludeGlobs = mkOption { ++ type = types.listOf types.str; ++ description = '' ++ List files matching a pattern to include. Uses Go blob pattern. ++ ''; ++ default = [ ]; ++ }; ++ ++ user = mkOption { ++ type = types.str; ++ description = '' ++ User owning the certs. ++ ''; ++ default = "acme"; ++ }; ++ }; ++ ++ serviceOpts = { ++ serviceConfig = { ++ User = cfg.user; ++ ExecStart = '' ++ ${lib.getExe pkgs.prometheus-node-cert-exporter} \ ++ --listen ${toString cfg.listenAddress}:${toString cfg.port} \ ++ --path ${concatStringsSep "," cfg.paths} \ ++ --exclude-path "${concatStringsSep "," cfg.excludePaths}" \ ++ --include-glob "${concatStringsSep "," cfg.includeGlobs}" \ ++ --exclude-glob "${concatStringsSep "," cfg.excludeGlobs}" \ ++ ${concatStringsSep " \\\n " cfg.extraFlags} ++ ''; ++ }; ++ }; ++} +diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix +index c15a3fd20b021..f59d61e69b92e 100644 +--- a/nixos/tests/prometheus-exporters.nix ++++ b/nixos/tests/prometheus-exporters.nix +@@ -1002,6 +1002,49 @@ let + ''; + }; + ++ node-cert = { ++ nodeName = "node_cert"; ++ exporterConfig = { ++ enable = true; ++ paths = ["/run/certs"]; ++ }; ++ exporterTest = '' ++ wait_for_unit("prometheus-node-cert-exporter.service") ++ wait_for_open_port(9141) ++ wait_until_succeeds( ++ "curl -sSf http://localhost:9141/metrics | grep 'ssl_certificate_expiry_seconds{.\\+path=\"/run/certs/node-cert\\.cert\".\\+}'" ++ ) ++ ''; ++ ++ metricProvider = { ++ system.activationScripts.cert.text = '' ++ mkdir -p /run/certs ++ cd /run/certs ++ ++ cat >ca.template <