update suite exceptions and links (deepchecks#706)

* Update utm source in suite example link * Update exception message of suite when not enough parameters for a check * Update exception message when model doesn't fit BasicModel * Fix tests * Update hpyerlink phrasing * Update all links to use Read more * Update links * Fix tests Co-authored-by: Itay Gabbay <[email protected]>
harveyyeung · Jan 20, 2022 · ed57d2e · ed57d2e
1 parent 6d37107
commit ed57d2e
Show file tree

Hide file tree

Showing 8 changed files with 37 additions and 33 deletions.
diff --git a/deepchecks/base/display_suite.py b/deepchecks/base/display_suite.py
@@ -295,7 +295,7 @@ def display_suite_result(suite_name: str, results: List[Union[CheckResult, Check
 
     suite_creation_example_link = (
         'https://docs.deepchecks.com/en/stable/examples/guides/create_a_custom_suite.html'
-        '?utm_source=suite_output&utm_medium=referral&utm_campaign=display_link'
+        '?utm_source=display_output&utm_medium=referral&utm_campaign=suite_link'
     )
 
     # suite summary
@@ -305,8 +305,8 @@ def display_suite_result(suite_name: str, results: List[Union[CheckResult, Check
             {prologue}<br>
             Each check may contain conditions (which will result in pass / fail / warning, represented by {icons})
             as well as other outputs such as plots or tables.<br>
-            Suites, checks and conditions can all be modified (see the
-            <a href={suite_creation_example_link} target="_blank">Create a Custom Suite</a> tutorial).
+            Suites, checks and conditions can all be modified. Read more about
+            <a href={suite_creation_example_link} target="_blank">custom suites</a>.
         </p>
         """
 

diff --git a/deepchecks/base/suite.py b/deepchecks/base/suite.py
@@ -192,7 +192,8 @@ def run(
                                                  model=model)
                         results.append(check_result)
                     else:
-                        results.append(Suite._get_unsupported_failure(check))
+                        msg = 'Check is irrelevant if not supplied with both train and test datasets'
+                        results.append(Suite._get_unsupported_failure(check, msg))
                 elif isinstance(check, SingleDatasetBaseCheck):
                     if train_dataset is not None:
                         # In case of train & test, doesn't want to skip test if train fails. so have to explicitly
@@ -211,13 +212,15 @@ def run(
                             check_result = CheckFailure(check, exp, ' - Test Dataset')
                         results.append(check_result)
                     if train_dataset is None and test_dataset is None:
-                        results.append(Suite._get_unsupported_failure(check))
+                        msg = 'Check is irrelevant if dataset is not supplied'
+                        results.append(Suite._get_unsupported_failure(check, msg))
                 elif isinstance(check, ModelOnlyBaseCheck):
                     if model is not None:
                         check_result = check.run(model=model)
                         results.append(check_result)
                     else:
-                        results.append(Suite._get_unsupported_failure(check))
+                        msg = 'Check is irrelevant if model is not supplied'
+                        results.append(Suite._get_unsupported_failure(check, msg))
                 else:
                     raise TypeError(f'Don\'t know how to handle type {check.__class__.__name__} in suite.')
             except Exception as exp:
@@ -228,8 +231,7 @@ def run(
         return SuiteResult(self.name, results)
 
     @classmethod
-    def _get_unsupported_failure(cls, check):
-        msg = 'Check is not supported for parameters given to suite'
+    def _get_unsupported_failure(cls, check, msg):
         return CheckFailure(check, DeepchecksNotSupportedError(msg))
 
 

diff --git a/deepchecks/checks/methodology/single_feature_contribution.py b/deepchecks/checks/methodology/single_feature_contribution.py
@@ -25,7 +25,7 @@
 
 pps_url = 'https://docs.deepchecks.com/en/stable/examples/checks/methodology/single_feature_contribution_train_test' \
           '.html?utm_source=display_output&utm_medium=referral&utm_campaign=check_link'
-pps_html_url = f'<a href={pps_url} target="_blank">Predictive Power Score</a>'
+pps_html = f'<a href={pps_url} target="_blank">Predictive Power Score</a>'
 
 
 class SingleFeatureContribution(SingleDatasetBaseCheck):
@@ -83,10 +83,12 @@ def plot(n_show_top=self.n_show_top):
             # Create graph:
             create_colorbar_barchart_for_check(x=top_to_show.index, y=top_to_show.values)
 
-        text = ['The PPS represents the ability of a feature to single-handedly predict another feature or label.',
-                'A high PPS (close to 1) can mean that this feature\'s success in predicting the label is'
-                ' actually due to data',
-                'leakage - meaning that the feature holds information that is based on the label to begin with.']
+        text = [
+            'The Predictive Power Score (PPS) is used to estimate the ability of a feature to predict the '
+            f'label by itself. (Read more about {pps_html})'
+            'A high PPS (close to 1) can mean that this feature\'s success in predicting the label is'
+            ' actually due to data leakage - meaning that the feature holds information that is based on the label '
+            'to begin with.']
 
         # display only if not all scores are 0
         display = [plot, *text] if s_ppscore.sum() else None
@@ -113,5 +115,5 @@ def condition(value: t.Dict[Hashable, float]) -> ConditionResult:
             else:
                 return ConditionResult(True)
 
-        return self.add_condition(f'Features\' {pps_html_url} (PPS) is not greater than {format_number(threshold)}',
+        return self.add_condition(f'Features\' Predictive Power Score is not greater than {format_number(threshold)}',
                                   condition)
diff --git a/deepchecks/checks/methodology/single_feature_contribution_train_test.py b/deepchecks/checks/methodology/single_feature_contribution_train_test.py
@@ -25,7 +25,7 @@
 
 pps_url = 'https://docs.deepchecks.com/en/stable/examples/checks/methodology/single_feature_contribution_train_test' \
           '.html?utm_source=display_output&utm_medium=referral&utm_campaign=check_link'
-pps_html_url = f'<a href={pps_url} target="_blank">Predictive Power Score</a>'
+pps_html = f'<a href={pps_url} target="_blank">Predictive Power Score</a>'
 
 
 class SingleFeatureContributionTrainTest(TrainTestBaseCheck):
@@ -122,7 +122,8 @@ def _single_feature_contribution_train_test(self, train_dataset: Dataset, test_d
         )
 
         text = [
-            f'The PPS ({pps_html_url}) is used to estimate the ability of a feature to predict the label by itself.'
+            'The Predictive Power Score (PPS) is used to estimate the ability of a feature to predict the '
+            f'label by itself. (Read more about {pps_html})'
             '',
             '<u>In the graph above</u>, we should suspect we have problems in our data if:',
             ''
@@ -169,7 +170,7 @@ def condition(value: t.Dict[Hashable, t.Dict[Hashable, float]]) -> ConditionResu
             else:
                 return ConditionResult(True)
 
-        return self.add_condition(f'Train-Test features\' {pps_html_url} (PPS) difference is not greater than '
+        return self.add_condition(f'Train-Test features\' Predictive Power Score difference is not greater than '
                                   f'{format_number(threshold)}', condition)
 
     def add_condition_feature_pps_in_train_not_greater_than(self: FC, threshold: float = 0.7) -> FC:
@@ -194,6 +195,5 @@ def condition(value: t.Dict[Hashable, t.Dict[Hashable, float]]) -> ConditionResu
             else:
                 return ConditionResult(True)
 
-        return \
-            self.add_condition(f'Train features\' {pps_html_url} (PPS) is not greater than {format_number(threshold)}',
-                               condition)
+        return self.add_condition(f'Train features\' Predictive Power Score is not greater than '
+                                  f'{format_number(threshold)}', condition)
diff --git a/deepchecks/utils/validation.py b/deepchecks/utils/validation.py
@@ -24,6 +24,10 @@
     'ensure_dataframe_type'
 ]
 
+supported_models_link = ('https://docs.deepchecks.com/en/stable/user-guide/supported_models.html'
+                         '?utm_source=display_output&utm_medium=referral&utm_campaign=exception_link')
+supported_models_html = f'<a href="{supported_models_link}" target="_blank">supported model types</a>'
+
 
 def model_type_validation(model: t.Any):
     """Receive any object and check if it's an instance of a model we support.
@@ -33,9 +37,7 @@ def model_type_validation(model: t.Any):
     """
     if not isinstance(model, BasicModel):
         raise errors.ModelValidationError(
-            'Model must be a structural subtype of the '
-            '`deepchecks.utils.typing.BasicModel` protocol.'
-            f'(received type: {type(model).__name__}'
+            f'Model supplied does not meets the minimal interface requirements. Read more about {supported_models_html}'
         )
 
 

diff --git a/tests/checks/methodology/single_feature_contribution_test.py b/tests/checks/methodology/single_feature_contribution_test.py
@@ -15,7 +15,6 @@
 
 from deepchecks import Dataset
 from deepchecks.checks.methodology import SingleFeatureContribution, SingleFeatureContributionTrainTest
-from deepchecks.checks.methodology.single_feature_contribution import pps_html_url
 from deepchecks.errors import DeepchecksValueError, DatasetValidationError
 
 from tests.checks.utils import equal_condition_result
@@ -142,7 +141,7 @@ def test_all_features_pps_upper_bound_condition_that_should_not_pass():
     # Assert
     assert_that(condition_result, equal_condition_result(
         is_pass=False,
-        name=f'Features\' {pps_html_url} (PPS) is not greater than {condition_value}',
+        name=f'Features\' Predictive Power Score is not greater than {condition_value}',
         details='Features with PPS above threshold: {\'x2\': \'0.84\', \'x4\': \'0.53\', \'x5\': \'0.42\'}'
     ))
 
@@ -160,7 +159,7 @@ def test_all_features_pps_upper_bound_condition_that_should_pass():
     # Assert
     assert_that(condition_result, equal_condition_result(
         is_pass=True,
-        name=f'Features\' {pps_html_url} (PPS) is not greater than {condition_value}',
+        name=f'Features\' Predictive Power Score is not greater than {condition_value}',
     ))
 
 
@@ -179,7 +178,7 @@ def test_train_test_condition_pps_difference_pass():
     # Assert
     assert_that(condition_result, equal_condition_result(
         is_pass=True,
-        name=f'Train-Test features\' {pps_html_url} (PPS) difference is not greater than {condition_value}'
+        name=f'Train-Test features\' Predictive Power Score difference is not greater than {condition_value}'
     ))
 
 
@@ -197,7 +196,7 @@ def test_train_test_condition_pps_difference_fail():
     # Assert
     assert_that(condition_result, equal_condition_result(
         is_pass=False,
-        name=f'Train-Test features\' {pps_html_url} (PPS) difference is not greater than {condition_value}',
+        name=f'Train-Test features\' Predictive Power Score difference is not greater than {condition_value}',
         details='Features with PPS difference above threshold: {\'x2\': \'0.31\'}'
     ))
 
@@ -217,7 +216,7 @@ def test_train_test_condition_pps_train_pass():
     # Assert
     assert_that(condition_result, equal_condition_result(
         is_pass=True,
-        name=f'Train features\' {pps_html_url} (PPS) is not greater than {condition_value}'
+        name=f'Train features\' Predictive Power Score is not greater than {condition_value}'
     ))
 
 
@@ -235,7 +234,7 @@ def test_train_test_condition_pps_train_fail():
     # Assert
     assert_that(condition_result, equal_condition_result(
         is_pass=False,
-        name=f'Train features\' {pps_html_url} (PPS) is not greater than {condition_value}',
+        name=f'Train features\' Predictive Power Score is not greater than {condition_value}',
         details='Features in train dataset with PPS above threshold: {\'x2\': \'0.84\'}'
     ))
 
diff --git a/tests/checks/overview/model_info_test.py b/tests/checks/overview/model_info_test.py
@@ -57,5 +57,5 @@ def test_model_info_wrong_input():
         calling(ModelInfo().run).with_args('some string'),
         raises(
             ModelValidationError, 
-            r'Model must be a structural subtype of the .*')
+            r'Model supplied does not meets the minimal interface requirements. Read more about .*')
     )
diff --git a/tests/checks/performance/performance_report_test.py b/tests/checks/performance/performance_report_test.py
@@ -41,8 +41,7 @@ def test_model_wrong_input(iris_labeled_dataset):
         calling(PerformanceReport().run).with_args(iris_labeled_dataset, iris_labeled_dataset,bad_model),
         raises(
             ModelValidationError, 
-            r'Model must be a structural subtype of the '
-            r'`deepchecks.utils.typing.BasicModel` protocol\. *')
+            r'Model supplied does not meets the minimal interface requirements. Read more about .*')
     )