Test improvements

tests/django_tests/manage_tests/runtests.py

@@ -0,0 +1,204 @@
+import argparse
+import json
+import os
+import shutil
+import importlib.util
+import subprocess
+import sys
+from json import JSONDecodeError
+
+MANAGE_DIR = os.path.dirname(os.path.realpath(__file__))
+DJANGO_TESTS_ROOT = os.path.dirname(MANAGE_DIR)
+
+TEST_DIR = os.path.join(
+    DJANGO_TESTS_ROOT,
+    'tests',
+    'v21',
+    'tests')
+
+django_version = None
+
+PARSER_ARGS = {
+    '--start-index': {
+        'default': None,
+        'type': int,
+        'dest': 'start_index'
+    },
+    '--django-version': {
+        'default': '21',
+        'type': str,
+        'dest': 'django_version'
+    },
+    '--run-currently-passing': {
+        'action': 'store_true',
+        'dest': 'run_currently_passing'
+    },
+    '--discover-passing': {
+        'action': 'store_true',
+        'dest': 'discover_passing'
+    },
+    '--discover-tests': {
+        'action': 'store_true',
+        'dest': 'discover_tests'
+    },
+}
+
+DEFAULT_TESTRUNNER_ARGS = {
+    'settings': '--settings=test_mongodb',
+    'failfast': '--failfast',
+    'parallel': '--parallel'
+}
+
+
+def check_settings():
+    if not os.path.isfile(os.path.join(TEST_DIR, 'test_mongodb.py')):
+        settings_path = os.path.join(MANAGE_DIR, 'settings', 'test_mongodb.py')
+        shutil.copy(settings_path, TEST_DIR)
+
+
+def get_django_parser():
+    spec = importlib.util.spec_from_file_location('runtests', os.path.join(TEST_DIR, 'runtests.py'))
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module.parser
+
+
+def extract_useful_args(args: list):
+    ret = []
+    for arg in args:
+        for parser_arg in PARSER_ARGS.keys():
+            if arg.startswith(parser_arg):
+                break
+        else:
+            ret.append(arg)
+    return ret
+
+
+def build_args(args: list, parsed_args):
+    uargs = extract_useful_args(args)
+
+    for option in DEFAULT_TESTRUNNER_ARGS:
+        if not getattr(parsed_args, option, False):
+            uargs.append(DEFAULT_TESTRUNNER_ARGS[option])
+
+    path = os.path.join(TEST_DIR, 'runtests.py')
+    return [path, 'test_name'] + uargs
+
+
+def get_file_contents():
+    try:
+        with open(os.path.join(MANAGE_DIR, 'tests_list.json'), 'r') as fp:
+            file_contents = json.load(fp)
+
+    except FileNotFoundError:
+        with open(os.path.join(MANAGE_DIR, 'tests_list.json'), 'x') as _:
+            pass
+        file_contents = {}
+
+    except JSONDecodeError:
+        file_contents = {}
+
+    return file_contents
+
+
+def get_tests_list():
+    file_contents = get_file_contents()
+
+    try:
+        test_list = file_contents[django_version]
+    except KeyError:
+        test_list = {
+            'all_tests': [],
+            'failing_tests': []
+        }
+    return test_list
+
+
+def dump_test_list(test_list):
+    file_contents = get_file_contents()
+    file_contents[django_version] = test_list
+
+    with open(os.path.join(MANAGE_DIR, 'tests_list.json'), 'w') as fp:
+        json.dump(file_contents, fp)
+
+
+def discover_tests():
+    dirs = os.listdir(TEST_DIR)
+    for i, adir in enumerate(dirs[:]):
+        if (adir.endswith('.py')
+                or adir.endswith('coveragerc')
+                or adir.endswith('__')
+                or adir.endswith('.rst')
+        ):
+            dirs.pop(i)
+
+    tests = get_tests_list()
+    tests['all_tests'] = dirs
+    dump_test_list(tests)
+
+
+def discover_passing(_parsed):
+    tests = get_tests_list()
+    orig_args = sys.argv
+    sys.argv = build_args(orig_args[1:], _parsed)
+    currently_failing = []
+
+    for i, atest in enumerate(tests['all_tests']):
+        sys.argv[1] = atest
+        print(f'## Executing test: {atest} no: {i} ##\n')
+        o = subprocess.run((['python'] + sys.argv))
+        if o.returncode != 0:
+            currently_failing.append(atest)
+
+    sys.argv = orig_args
+    tests['failing_tests'] = currently_failing
+    dump_test_list(tests)
+
+
+def check_passing(_parsed):
+    tests = get_tests_list()
+    orig_args = sys.argv
+    sys.argv = build_args(orig_args[1:], _parsed)
+    passing = set(tests['all_tests']) - set(tests['failing_tests'])
+    pass_exit_code = 0
+    fail_exit_code = 1
+
+    for i, atest in enumerate(passing):
+        sys.argv[1] = atest
+        print(f'## Executing test: {atest} no: {i} ##\n')
+        o = subprocess.run((['python'] + sys.argv))
+        if o.returncode != 0:
+            sys.argv = orig_args
+            return fail_exit_code
+        print(f'## Ran test: {atest}##\n')
+
+    sys.argv = orig_args
+    return pass_exit_code
+
+
+def get_parser():
+    _parser = argparse.ArgumentParser(parents=[get_django_parser()], add_help=False)
+    for option, arg in PARSER_ARGS.items():
+        _parser.add_argument(option, **arg)
+
+    return _parser
+
+
+if __name__ == '__main__':
+    parser = get_parser()
+    parsed = parser.parse_args()
+    django_version = 'v' + parsed.django_version
+
+    TEST_DIR = os.path.join(
+        DJANGO_TESTS_ROOT,
+        'tests',
+        django_version,
+        'tests')
+    check_settings()
+
+    if parsed.discover_tests:
+        discover_tests()
+    if parsed.discover_passing:
+        discover_passing(parsed)
+    if parsed.run_currently_passing:
+        exit(check_passing(parsed))

tests/django_tests/manage_tests/tests_list.json

@@ -0,0 +1 @@
+{"v21": {"all_tests": [".coveragerc", "absolute_url_overrides", "admin_autodiscover", "admin_changelist", "admin_checks", "admin_custom_urls", "admin_default_site", "admin_docs", "admin_filters", "admin_inlines", "admin_ordering", "admin_registration", "admin_scripts", "admin_utils", "admin_views", "admin_widgets", "aggregation", "aggregation_regress", "annotations", "apps", "app_loading", "auth_tests", "backends", "base", "bash_completion", "basic", "builtin_server", "bulk_create", "cache", "check_framework", "choices", "conditional_processing", "contenttypes_tests", "context_processors", "csrf_tests", "custom_columns", "custom_lookups", "custom_managers", "custom_methods", "custom_migration_operations", "custom_pk", "datatypes", "dates", "datetimes", "dbshell", "db_functions", "db_typecasts", "db_utils", "decorators", "defer", "defer_regress", "delete", "delete_regress", "deprecation", "dispatch", "distinct_on_fields", "empty", "expressions", "expressions_case", "expressions_window", "extra_regress", "field_deconstruction", "field_defaults", "field_subclassing", "files", "file_storage", "file_uploads", "filtered_relation", "fixtures", "fixtures_model_package", "fixtures_regress", "flatpages_tests", "force_insert_update", "foreign_object", "forms_tests", "from_db_value", "generic_inline_admin", "generic_relations", "generic_relations_regress", "generic_views", "get_earliest_or_latest", "get_object_or_404", "get_or_create", "gis_tests", "handlers", "httpwrappers", "humanize_tests", "i18n", "import_error_package", "indexes", "inline_formsets", "inspectdb", "introspection", "invalid_models_tests", "known_related_objects", "logging_tests", "lookup", "m2m_and_m2o", "m2m_intermediary", "m2m_multiple", "m2m_recursive", "m2m_regress", "m2m_signals", "m2m_through", "m2m_through_regress", "m2o_recursive", "mail", "managers_regress", "many_to_many", "many_to_one", "many_to_one_null", "max_lengths", "messages_tests", "middleware", "middleware_exceptions", "migrate_signals", "migrations", "migrations2", "migration_test_data_persistence", "modeladmin", "model_fields", "model_forms", "model_formsets", "model_formsets_regress", "model_indexes", "model_inheritance", "model_inheritance_regress", "model_meta", "model_options", "model_package", "model_regress", "multiple_database", "mutually_referential", "nested_foreign_keys", "no_models", "null_fk", "null_fk_ordering", "null_queries", "one_to_one", "ordering", "order_with_respect_to", "or_lookups", "pagination", "postgres_tests", "prefetch_related", "project_template", "properties", "proxy_models", "proxy_model_inheritance", "queries", "queryset_pickle", "raw_query", "README.rst", "redirects_tests", "requests", "requirements", "reserved_names", "resolve_url", "responses", "reverse_lookup", "runtests.py", "runtests_wrapper.del.py", "save_delete_hooks", "schema", "select_for_update", "select_related", "select_related_onetoone", "select_related_regress", "serializers", "servers", "sessions_tests", "settings_tests", "settings_test_djongo.del.py", "shell", "shortcuts", "signals", "signed_cookies_tests", "signing", "sitemaps_tests", "sites_framework", "sites_tests", "staticfiles_tests", "str", "string_lookup", "swappable_models", "syndication_tests", "templates", "template_backends", "template_loader", "template_tests", "test_client", "test_client_regress", "test_exceptions", "test_mongodb.py", "test_runner", "test_runner_apps", "test_sqlite.py", "test_utils", "timezones", "transactions", "transaction_hooks", "unmanaged_models", "update", "update_only_fields", "urlpatterns", "urlpatterns_reverse", "urls.py", "user_commands", "utils_tests", "validation", "validators", "version", "view_tests", "wsgi", "__pycache__"], "failing_tests": [".coveragerc", "admin_changelist", "admin_checks", "admin_filters", "admin_inlines", "admin_ordering", "admin_scripts", "admin_utils", "admin_views", "admin_widgets", "aggregation", "aggregation_regress", "annotations", "apps", "auth_tests", "backends", "basic", "bulk_create", "cache", "contenttypes_tests", "custom_columns", "custom_lookups", "custom_managers", "custom_methods", "custom_pk", "datatypes", "dates", "datetimes", "db_functions", "defer", "delete", "delete_regress", "expressions", "expressions_case", "extra_regress", "filtered_relation", "fixtures", "fixtures_regress", "force_insert_update", "foreign_object", "forms_tests", "generic_relations", "generic_relations_regress", "generic_views", "get_earliest_or_latest", "get_or_create", "gis_tests", "handlers", "i18n", "import_error_package", "inspectdb", "introspection", "invalid_models_tests", "lookup", "m2m_through", "m2m_through_regress", "mail", "many_to_many", "many_to_one", "max_lengths", "migrations", "modeladmin", "model_fields", "model_forms", "model_formsets", "model_indexes", "model_inheritance", "model_inheritance_regress", "model_regress", "multiple_database", "null_queries", "one_to_one", "ordering", "order_with_respect_to", "or_lookups", "prefetch_related", "properties", "proxy_models", "queries", "queryset_pickle", "raw_query", "README.rst", "reserved_names", "runtests.py", "runtests_wrapper.del.py", "schema", "select_related", "select_related_onetoone", "select_related_regress", "serializers", "servers", "sessions_tests", "settings_test_djongo.del.py", "sitemaps_tests", "staticfiles_tests", "syndication_tests", "template_tests", "test_client", "test_mongodb.py", "test_runner", "test_sqlite.py", "test_utils", "timezones", "transactions", "transaction_hooks", "update", "urls.py", "user_commands", "utils_tests", "validation", "validators", "view_tests"]}, "v22": {"all_tests": [".coveragerc", "absolute_url_overrides", "admin_autodiscover", "admin_changelist", "admin_checks", "admin_custom_urls", "admin_default_site", "admin_docs", "admin_filters", "admin_inlines", "admin_ordering", "admin_registration", "admin_scripts", "admin_utils", "admin_views", "admin_widgets", "aggregation", "aggregation_regress", "annotations", "apps", "app_loading", "auth_tests", "backends", "base", "bash_completion", "basic", "builtin_server", "bulk_create", "cache", "check_framework", "choices", "conditional_processing", "constraints", "contenttypes_tests", "context_processors", "csrf_tests", "custom_columns", "custom_lookups", "custom_managers", "custom_methods", "custom_migration_operations", "custom_pk", "datatypes", "dates", "datetimes", "dbshell", "db_functions", "db_typecasts", "db_utils", "decorators", "defer", "defer_regress", "delete", "delete_regress", "deprecation", "dispatch", "distinct_on_fields", "empty", "expressions", "expressions_case", "expressions_window", "extra_regress", "field_deconstruction", "field_defaults", "field_subclassing", "files", "file_storage", "file_uploads", "filtered_relation", "fixtures", "fixtures_model_package", "fixtures_regress", "flatpages_tests", "force_insert_update", "foreign_object", "forms_tests", "from_db_value", "generic_inline_admin", "generic_relations", "generic_relations_regress", "generic_views", "get_earliest_or_latest", "get_object_or_404", "get_or_create", "gis_tests", "handlers", "httpwrappers", "humanize_tests", "i18n", "import_error_package", "indexes", "inline_formsets", "inspectdb", "introspection", "invalid_models_tests", "known_related_objects", "logging_tests", "lookup", "m2m_and_m2o", "m2m_intermediary", "m2m_multiple", "m2m_recursive", "m2m_regress", "m2m_signals", "m2m_through", "m2m_through_regress", "m2o_recursive", "mail", "managers_regress", "many_to_many", "many_to_one", "many_to_one_null", "max_lengths", "messages_tests", "middleware", "middleware_exceptions", "migrate_signals", "migrations", "migrations2", "migration_test_data_persistence", "modeladmin", "model_fields", "model_forms", "model_formsets", "model_formsets_regress", "model_indexes", "model_inheritance", "model_inheritance_regress", "model_meta", "model_options", "model_package", "model_regress", "multiple_database", "mutually_referential", "nested_foreign_keys", "no_models", "null_fk", "null_fk_ordering", "null_queries", "one_to_one", "ordering", "order_with_respect_to", "or_lookups", "pagination", "postgres_tests", "prefetch_related", "project_template", "properties", "proxy_models", "proxy_model_inheritance", "queries", "queryset_pickle", "raw_query", "README.rst", "redirects_tests", "requests", "requirements", "reserved_names", "resolve_url", "responses", "reverse_lookup", "runtests.py", "save_delete_hooks", "schema", "select_for_update", "select_related", "select_related_onetoone", "select_related_regress", "serializers", "servers", "sessions_tests", "settings_tests", "shell", "shortcuts", "signals", "signed_cookies_tests", "signing", "sitemaps_tests", "sites_framework", "sites_tests", "staticfiles_tests", "str", "string_lookup", "swappable_models", "syndication_tests", "templates", "template_backends", "template_loader", "template_tests", "test_client", "test_client_regress", "test_exceptions", "test_mongodb.py", "test_runner", "test_runner_apps", "test_sqlite.py", "test_utils", "timezones", "transactions", "transaction_hooks", "unmanaged_models", "update", "update_only_fields", "urlpatterns", "urlpatterns_reverse", "urls.py", "user_commands", "utils_tests", "validation", "validators", "version", "view_tests", "wsgi"], "failing_tests": [".coveragerc", "admin_changelist", "admin_filters", "admin_ordering", "admin_utils", "admin_views", "aggregation", "aggregation_regress", "annotations", "auth_tests", "backends", "basic", "bulk_create", "cache", "constraints", "contenttypes_tests", "custom_columns", "custom_lookups", "custom_managers", "custom_methods", "custom_pk", "datatypes", "dates", "datetimes", "db_functions", "defer", "delete", "delete_regress", "expressions", "expressions_case", "extra_regress", "filtered_relation", "fixtures", "fixtures_regress", "force_insert_update", "foreign_object", "generic_relations", "generic_relations_regress", "generic_views", "get_earliest_or_latest", "get_or_create", "gis_tests", "i18n", "import_error_package", "indexes", "inspectdb", "introspection", "lookup", "m2m_through", "m2m_through_regress", "many_to_many", "many_to_one", "migrations", "modeladmin", "model_fields", "model_forms", "model_formsets", "model_inheritance", "model_inheritance_regress", "model_regress", "multiple_database", "null_queries", "one_to_one", "ordering", "order_with_respect_to", "or_lookups", "prefetch_related", "proxy_models", "queries", "queryset_pickle", "raw_query", "README.rst", "reserved_names", "runtests.py", "schema", "select_related", "select_related_onetoone", "select_related_regress", "serializers", "sitemaps_tests", "syndication_tests", "test_mongodb.py", "test_runner", "test_sqlite.py", "test_utils", "timezones", "transactions", "transaction_hooks", "update", "urls.py", "validation", "view_tests"]}}