Skip two flaky tests and tweak more test timeouts (pantsbuild#8687)

### Skip some exception sink integration tests on macOS
These shards have chronically flaked by hanging since at least July pantsbuild#8127. They are our most egregious Python flake.

We will still test these four tests on Linux and only skip them on macOS, as this is a macOS specific 
issue.

### Skip `remote::tests::dropped_request_cancels`
This seems to flake roughly 40% of the time pantsbuild#8405. It is our most egregious Rust flake.

### Tweak some other tests
We mark some other tests as flaky and bump their timeouts as relevant to hopefully stabilize CI further.

src/rust/engine/process_execution/src/remote_tests.rs

@@ -1081,6 +1081,7 @@ fn timeout_after_sufficiently_delayed_getoperations() {
 }
 
 #[test]
+#[ignore] // https://github.com/pantsbuild/pants/issues/8405
 fn dropped_request_cancels() {
   let request_timeout = Duration::new(10, 0);
   let delayed_operation_time = Duration::new(5, 0);

tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/BUILD

@@ -22,7 +22,7 @@ python_tests(
     'testprojects/src/scala/org/pantsbuild/testproject:javasources_directory',
     'testprojects/src/scala/org/pantsbuild/testproject:mutual_directory',
   ],
-  timeout = 600,
+  timeout = 660,
   tags = {'integration'},
 )
 
@@ -35,7 +35,7 @@ python_tests(
     'testprojects/src/scala/org/pantsbuild/testproject:mutual_directory',
     'testprojects/src/scala/org/pantsbuild/testproject:public_inference_directory',
   ],
-  timeout = 600,
+  timeout = 800,
   tags = {'integration'},
 )
 

tests/python/pants_test/backend/jvm/tasks/jvm_compile/rsc/test_rsc_compile_integration_youtline.py

@@ -1,6 +1,8 @@
 # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import pytest
+
 from pants_test.backend.jvm.tasks.jvm_compile.rsc.rsc_compile_integration_base import (
   RscCompileIntegrationBase,
   ensure_compile_rsc_execution_strategy,
@@ -13,6 +15,7 @@ class RscCompileIntegrationYoutline(RscCompileIntegrationBase):
   def test_basic_binary(self):
     self._testproject_compile("mutual", "bin", "A")
 
+  @pytest.mark.flaky(retries=1)
   @ensure_compile_rsc_execution_strategy(RscCompileIntegrationBase.outline_and_zinc)
   def test_public_inference(self):
     self._testproject_compile("public_inference", "public_inference", "PublicInference")

tests/python/pants_test/backend/jvm/tasks/jvm_compile/scala/BUILD

@@ -9,6 +9,6 @@ python_tests(
     'tests/python/pants_test/backend/jvm/tasks/jvm_compile:base_compile_integration_test',
     'examples/src/scala/org/pantsbuild/example:scalac_directory',
   ],
-  timeout = 600,
+  timeout = 640,
   tags = {'integration'},
 )

tests/python/pants_test/backend/python/BUILD

@@ -49,6 +49,7 @@ python_tests(
     'testprojects:pants_plugins_directory',
   ],
   tags={'integration'},
+  timeout = 120,
 )
 
 python_library(

tests/python/pants_test/base/BUILD

@@ -31,7 +31,7 @@ python_tests(
     'testprojects/src/java/org/pantsbuild/testproject:phrases_directory',
   ],
   tags = {'integration'},
-  timeout = 500,
+  timeout = 530,
 )
 
 python_tests(

tests/python/pants_test/base/test_exception_sink_integration.py

@@ -6,12 +6,14 @@
 import time
 from contextlib import contextmanager
 from textwrap import dedent
+from unittest import skipIf
 
 from pants.base.build_environment import get_buildroot
 from pants.base.exception_sink import ExceptionSink
 from pants.testutil.pants_run_integration_test import PantsRunIntegrationTest
 from pants.util.contextutil import environment_as, temporary_dir
 from pants.util.dirutil import read_file, safe_file_dump, safe_mkdir, touch
+from pants.util.osutil import get_normalized_os_name
 
 
 class ExceptionSinkIntegrationTest(PantsRunIntegrationTest):
@@ -150,6 +152,7 @@ def _send_signal_to_waiter_handle(self, signum):
       # Return the (failed) pants execution result.
       yield (workdir, waiter_run)
 
+  @skipIf(get_normalized_os_name() == "darwin", "https://github.com/pantsbuild/pants/issues/8127")
   def test_dumps_logs_on_signal(self):
     """Send signals which are handled, but don't get converted into a KeyboardInterrupt."""
     signal_names = {
@@ -169,6 +172,7 @@ def test_dumps_logs_on_signal(self):
         self._assert_graceful_signal_log_matches(
           waiter_run.pid, signum, signame, read_file(shared_log_file))
 
+  @skipIf(get_normalized_os_name() == "darwin", "https://github.com/pantsbuild/pants/issues/8127")
   def test_dumps_traceback_on_sigabrt(self):
     # SIGABRT sends a traceback to the log file for the current process thanks to
     # faulthandler.enable().
@@ -183,6 +187,7 @@ def test_dumps_traceback_on_sigabrt(self):
       # faulthandler.enable() only allows use of a single logging file at once for fatal tracebacks.
       self.assertEqual('', read_file(shared_log_file))
 
+  @skipIf(get_normalized_os_name() == "darwin", "https://github.com/pantsbuild/pants/issues/8127")
   def test_prints_traceback_on_sigusr2(self):
     with self._make_waiter_handle() as (workdir, pid, join):
       # Send SIGUSR2, then sleep so the signal handler from faulthandler.register() can run.
@@ -195,6 +200,7 @@ def test_prints_traceback_on_sigusr2(self):
 Current thread [^\n]+ \\(most recent call first\\):
 """)
 
+  @skipIf(get_normalized_os_name() == "darwin", "https://github.com/pantsbuild/pants/issues/8127")
   def test_keyboardinterrupt(self):
     with self._send_signal_to_waiter_handle(signal.SIGINT) as (_, waiter_run):
       self.assertIn(