[narwhal] (re)enable end to end tests for Narwhal (MystenLabs#6044)

Enable end to end nightly tests for Narwhal

.config/nextest.toml

@@ -11,5 +11,18 @@ retries = 5
 # Timeout tests after 4 minutes
 slow-timeout = { period = "60s", terminate-after = 4 }
 
+[profile.narwhalnightly]
+# Print out output for failing tests as soon as they fail, and also at the end
+# of the run (for easy scrollability).
+failure-output = "immediate-final"
+# Show skipped tests in the CI output.
+status-level = "skip"
+# Do not cancel the test run on the first failure.
+fail-fast = false
+# Retry failing tests in order to not block builds on flaky tests
+retries = 2
+# Mark tests as slow after 25mins, kill them after 50
+slow-timeout = { period = "1500s", terminate-after = 1 }
+
 [profile.ci.junit]
 path = "junit.xml"

.github/workflows/nightly-narwhal.yml

@@ -0,0 +1,87 @@
+name: nightly-narwhal
+
+on:
+  ## Allow triggering this workflow manually via GitHub CLI/web
+  workflow_dispatch:
+  schedule:
+    - cron: '0 3 * * *'  # every 03:00 every night
+
+env:
+  CARGO_TERM_COLOR: always
+  # Disable incremental compilation.
+  #
+  # Incremental compilation is useful as part of an edit-build-test-edit cycle,
+  # as it lets the compiler avoid recompiling code that hasn't changed. However,
+  # on CI, we're not making small edits; we're almost always building the entire
+  # project from scratch. Thus, incremental compilation on CI actually
+  # introduces *additional* overhead to support making future builds
+  # faster...but no future builds will ever occur in any given CI environment.
+  #
+  # See https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
+  # for details.
+  CARGO_INCREMENTAL: 0
+  # Allow more retries for network requests in cargo (downloading crates) and
+  # rustup (installing toolchains). This should help to reduce flaky CI failures
+  # from transient network timeouts or other issues.
+  CARGO_NET_RETRY: 10
+  RUSTUP_MAX_RETRIES: 10
+  # Don't emit giant backtraces in the CI logs.
+  RUST_BACKTRACE: short
+
+jobs:
+  end-to-end-tests:
+    name: Narwhal run long-running end-to-end tests
+    runs-on: ubuntu-ghcloud
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+      - uses: taiki-e/install-action@nextest
+      # Enable caching of the 'librocksdb-sys' crate by additionally caching the
+      # 'librocksdb-sys' src directory which is managed by cargo
+      - uses: bmwill/rust-cache@v1 # Fork of 'Swatinem/rust-cache' which allows caching additional paths
+        with:
+          path: ~/.cargo/registry/src/**/librocksdb-sys-*
+      # We run only the integration tests that are ignored under the primary package.
+      # These are considered the end-to-end "slow" tests for us
+      - name: cargo nextest
+        run: |
+          cargo nextest run -E 'kind(test) and package(narwhal-primary)' --profile narwhalnightly --run-ignored ignored-only
+
+  report-status:
+    name: Report Status
+    runs-on: ubuntu-latest
+    if: always()
+    needs: [end-to-end-tests]
+    steps:
+      - uses: technote-space/workflow-conclusion-action@v3
+
+      - name: Post to slack
+        uses: slackapi/[email protected]
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        if: env.WORKFLOW_CONCLUSION == 'failure' # notify only if failure
+        with:
+          channel-id: 'narwhal-tusk'
+          payload: |
+            {
+              "text": "NARWAL Workflow *${{ github.workflow }}* failed",
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "NARWHAL Workflow *${{ github.workflow }}* failed"
+                  }
+                },
+                {
+                  "type": "divider"
+                },
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "Logs are here: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+                  }
+                }
+              ]
+            }