loading: make __precompile__ the default (JuliaLang#26991)

fix JuliaLang#26282

NEWS.md

@@ -233,6 +233,10 @@ Language changes
     `size`, `length`, and `@inbounds`. To optionally enforce conventional indices,
     you can `@assert !has_offset_axes(A)`.
 
+  * Module pre-compilation is now the default for code loading. Adding a
+    `__precompile__()` declaration is no longer necessary, although
+    `__precompile__(false)` can still be used to opt-out ([#26991]).
+
 Breaking changes
 ----------------
 

base/loading.jl

@@ -744,46 +744,30 @@ function include_dependency(path::AbstractString)
     return nothing
 end
 
-# We throw PrecompilableError(true) when a module wants to be precompiled but isn't,
-# and PrecompilableError(false) when a module doesn't want to be precompiled but is
-struct PrecompilableError <: Exception
-    isprecompilable::Bool
-end
+# we throw PrecompilableError when a module doesn't want to be precompiled
+struct PrecompilableError <: Exception end
 function show(io::IO, ex::PrecompilableError)
-    if ex.isprecompilable
-        print(io, "Declaring __precompile__(true) is only allowed in module files being imported.")
-    else
-        print(io, "Declaring __precompile__(false) is not allowed in files that are being precompiled.")
-    end
+    print(io, "Declaring __precompile__(false) is not allowed in files that are being precompiled.")
 end
-precompilableerror(ex::PrecompilableError, c) = ex.isprecompilable == c
-precompilableerror(ex::WrappedException, c) = precompilableerror(ex.error, c)
-precompilableerror(ex, c) = false
+precompilableerror(ex::PrecompilableError) = true
+precompilableerror(ex::WrappedException) = precompilableerror(ex.error)
+precompilableerror(@nospecialize ex) = false
 
-# Call __precompile__ at the top of a file to force it to be precompiled (true), or
-# to be prevent it from being precompiled (false).  __precompile__(true) is
-# ignored except within "require" call.
+# Call __precompile__(false) at the top of a tile prevent it from being precompiled (false)
 """
-    __precompile__(isprecompilable::Bool=true)
-
-Specify whether the file calling this function is precompilable. If `isprecompilable` is
-`true`, then `__precompile__` throws an exception when the file is loaded by
-`using`/`import`/`require` *unless* the file is being precompiled, and in a module file it
-causes the module to be automatically precompiled when it is imported. Typically,
-`__precompile__()` should occur before the `module` declaration in the file.
+    __precompile__(false)
 
+Specify that the file calling this function is not precompilable.
 If a module or file is *not* safely precompilable, it should call `__precompile__(false)` in
 order to throw an error if Julia attempts to precompile it.
-
-`__precompile__()` should *not* be used in a module unless all of its dependencies are also
-using `__precompile__()`. Failure to do so can result in a runtime error when loading the module.
 """
-function __precompile__(isprecompilable::Bool=true)
-    if (JLOptions().use_compiled_modules != 0 &&
-        isprecompilable != (0 != ccall(:jl_generating_output, Cint, ())) &&
-        !(isprecompilable && toplevel_load[]))
-        throw(PrecompilableError(isprecompilable))
+@noinline function __precompile__(isprecompilable::Bool=true)
+    if isprecompilable
+        depwarn("__precompile__() is now the default", :__precompile__)
+    elseif 0 != ccall(:jl_generating_output, Cint, ())
+        throw(PrecompilableError())
     end
+    nothing
 end
 
 # require always works in Main scope and loads files from node 1
@@ -932,10 +916,9 @@ function _require(pkg::PkgId)
         end
 
         # attempt to load the module file via the precompile cache locations
-        doneprecompile = false
         if JLOptions().use_compiled_modules != 0
-            doneprecompile = _require_search_from_serialized(pkg, path)
-            if !isa(doneprecompile, Bool)
+            m = _require_search_from_serialized(pkg, path)
+            if !isa(m, Bool)
                 return
             end
         end
@@ -948,21 +931,24 @@ function _require(pkg::PkgId)
                      This may mean module $name does not support precompilation but is imported by a module that does."""
                 if JLOptions().incremental != 0
                     # during incremental precompilation, this should be fail-fast
-                    throw(PrecompilableError(false))
+                    throw(PrecompilableError())
                 end
             end
         end
 
-        if doneprecompile === true || JLOptions().incremental != 0
-            # spawn off a new incremental pre-compile task for recursive `require` calls
-            # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
-            cachefile = compilecache(pkg)
-            m = _require_from_serialized(cachefile)
-            if isa(m, Exception)
-                @warn "The call to compilecache failed to create a usable precompiled cache file for module $name" exception=m
-                # fall-through, TODO: disable __precompile__(true) error so that the normal include will succeed
-            else
-                return
+        if JLOptions().use_compiled_modules != 0
+            if (0 == ccall(:jl_generating_output, Cint, ())) || (JLOptions().incremental != 0)
+                # spawn off a new incremental pre-compile task for recursive `require` calls
+                # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
+                cachefile = compilecache(pkg)
+                m = _require_from_serialized(cachefile)
+                if isa(m, Exception)
+                    if !precompilableerror(m)
+                        @warn "The call to compilecache failed to create a usable precompiled cache file for module $name" exception=m
+                    end
+                else
+                    return
+                end
             end
         end
 
@@ -977,22 +963,6 @@ function _require(pkg::PkgId)
         try
             include_relative(__toplevel__, path)
             return
-        catch ex
-            if uuid !== old_uuid
-                ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, old_uuid)
-            end
-            if doneprecompile === true || JLOptions().use_compiled_modules == 0 || !precompilableerror(ex, true)
-                rethrow() # rethrow non-precompilable=true errors
-            end
-            # the file requested `__precompile__`, so try to build a cache file and use that
-            cachefile = compilecache(pkg)
-            m = _require_from_serialized(cachefile)
-            if isa(m, Exception)
-                @warn """Module `$name` declares `__precompile__(true)` but `require` failed
-                         to create a usable precompiled cache file""" exception=m
-                # TODO: disable __precompile__(true) error and do normal include instead of error
-                error("Module $name declares __precompile__(true) but require failed to create a usable precompiled cache file.")
-            end
         finally
             if uuid !== old_uuid
                 ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), __toplevel__, old_uuid)

doc/src/manual/modules.md

@@ -217,14 +217,14 @@ This prevents name conflicts for globals initialized after load time.
 ### Module initialization and precompilation
 
 Large modules can take several seconds to load because executing all of the statements in a module
-often involves compiling a large amount of code. Julia provides the ability to create precompiled
-versions of modules to reduce this time.
+often involves compiling a large amount of code.
+Julia creates precompiled caches of the module to reduce this time.
 
-To create an incremental precompiled module file, add `__precompile__()` at the top of your module
-file (before the `module` starts). This will cause it to be automatically compiled the first time
+The incremental precompiled module file are created and used automatically when using `import`
+or `using` to load a module.  This will cause it to be automatically compiled the first time
 it is imported. Alternatively, you can manually call `Base.compilecache(modulename)`. The resulting
 cache files will be stored in `DEPOT_PATH[1]/compiled/`. Subsequently, the module is automatically
-recompiled upon `import` whenever any of its dependencies change; dependencies are modules it
+recompiled upon `using` or `import` whenever any of its dependencies change; dependencies are modules it
 imports, the Julia build, files it includes, or explicit dependencies declared by `include_dependency(path)`
 in the module file(s).
 
@@ -240,20 +240,22 @@ incompatibilities between the running system and the precompile cache. If you wa
 to the source reflected in the running system, you should call `reload("Module")` on the module
 you changed, and any module that depended on it in which you want to see the change reflected.
 
-Precompiling a module also recursively precompiles any modules that are imported therein. If you
-know that it is *not* safe to precompile your module (for the reasons described below), you should
-put `__precompile__(false)` in the module file to cause `Base.compilecache` to throw an error
-(and thereby prevent the module from being imported by any other precompiled module).
-
-`__precompile__()` should *not* be used in a module unless all of its dependencies are also using
-`__precompile__()`. Failure to do so can result in a runtime error when loading the module.
-
-In order to make your module work with precompilation, however, you may need to change your module
-to explicitly separate any initialization steps that must occur at *runtime* from steps that can
-occur at *compile time*.  For this purpose, Julia allows you to define an `__init__()` function
-in your module that executes any initialization steps that must occur at runtime. This function
-will not be called during compilation (`--output-*` or `__precompile__()`). You may, of course,
-call it manually if necessary, but the default is to assume this function deals with computing
+If you know that a module is *not* safe to precompile your module
+(for example, for one of the reasons described below), you should
+put `__precompile__(false)` in the module file (typically placed at the top).
+This will cause `Base.compilecache` to throw an error, and will cause `using` / `import` to load it
+directly into the current process and skip the precompile and caching.
+This also thereby prevents the module from being imported by any other precompiled module.
+
+You may need to be aware of certain behaviors inherent in the creation of incremental shared libraries
+which may require care when writing your module. For example, external state is not preserved.
+To accommodate this, explicitly separate any initialization steps that must occur at *runtime*
+from steps that can occur at *compile time*.
+For this purpose, Julia allows you to define an `__init__()` function in your module that executes
+any initialization steps that must occur at runtime.
+This function will not be called during compilation (`--output-*`).
+Effectively, you can assume it will be run exactly once in the lifetime of the code.
+You may, of course, call it manually if necessary, but the default is to assume this function deals with computing
 state for the local machine, which does not need to be – or even should not be – captured
 in the compiled image. It will be called after the module is loaded into a process, including
 if it is being loaded into an incremental compile (`--output-incremental=yes`), but not if it
@@ -382,6 +384,5 @@ It is sometimes helpful during module development to turn off incremental precom
 command line flag `--compiled-modules={yes|no}` enables you to toggle module precompilation on and
 off. When Julia is started with `--compiled-modules=no` the serialized modules in the compile cache
 are ignored when loading modules and module dependencies. `Base.compilecache` can still be called
-manually and it will respect `__precompile__()` directives for the module. The state of this command
-line flag is passed to [`Pkg.build`] to disable automatic precompilation triggering when installing,
-updating, and explicitly building packages.
+manually. The state of this command line flag is passed to `Pkg.build` to disable automatic
+precompilation triggering when installing, updating, and explicitly building packages.

stdlib/Base64/src/Base64.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 module Base64
 
 using Base: has_offset_axes

stdlib/CRC32c/src/CRC32c.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
 Standard library module for computing the CRC-32c checksum.
 

stdlib/Dates/src/Dates.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
     Dates
 

stdlib/DelimitedFiles/src/DelimitedFiles.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
 Utilities for reading and writing delimited files, for example ".csv".
 See [`readdlm`](@ref) and [`writedlm`](@ref).

stdlib/Distributed/src/Distributed.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
 Tools for distributed parallel processing.
 """

stdlib/FileWatching/src/FileWatching.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
 Utilities for monitoring files and file descriptors for events.
 """

stdlib/Future/src/Future.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 "The `Future` module implements future behavior of already existing functions,
 which will replace the current version in a future release of Julia."
 module Future

stdlib/InteractiveUtils/src/InteractiveUtils.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 module InteractiveUtils
 
 export apropos, edit, less, code_warntype, code_llvm, code_native, methodswith, varinfo,

stdlib/LibGit2/src/LibGit2.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
 Interface to [libgit2](https://libgit2.github.com/).
 """

stdlib/Libdl/src/Libdl.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 module Libdl
 @doc """
 Interface to libdl. Provides dynamic linking support.

stdlib/LinearAlgebra/src/LinearAlgebra.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
 Linear algebra module. Provides array arithmetic,
 matrix factorizations and other linear algebra related

stdlib/Logging/src/Logging.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 module Logging
 
 # For now, simply import most names from Base - we don't want to fully

stdlib/Markdown/src/Markdown.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
 Tools for working with the Markdown file format. Mainly for documentation.
 """

stdlib/Mmap/src/Mmap.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
 Low level module for mmap (memory mapping of files).
 """

stdlib/OldPkg/src/OldPkg.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
     OldPkg
 

stdlib/Pkg/src/API.jl

@@ -465,9 +465,9 @@ function precompile(ctx::Context)
             end
         end
         if !found_matching_precompile
-            # Only precompile packages that has contains `__precompile__` or `__precompile__(true)`
+            # Don't bother attempting to precompile packages that appear to contain `__precompile__(false)`
             source = read(sourcepath, String)
-            if occursin(r"__precompile__\(\)|__precompile__\(true\)", source)
+            if !occursin(r"__precompile__\(false\)", source)
                 push!(needs_to_be_precompiled, pkg.name)
             end
         end

stdlib/Pkg/src/Pkg.jl

@@ -1,6 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
 module Pkg
 
 import Random

stdlib/Pkg/test/repl.jl

@@ -121,7 +121,10 @@ temp_pkg_dir() do project_path; cd(project_path) do; mktempdir() do tmp_pkg_path
     p2 = git_init_package(tmp_pkg_path, joinpath(@__DIR__, "test_packages/$pkg2"))
     Pkg.REPLMode.pkgstr("add $p2")
     Pkg.REPLMode.pkgstr("pin $pkg2")
-    @eval import $(Symbol(pkg2))
+    # FIXME: this confuses the precompile logic to know what is going on with the user
+    # FIXME: why isn't this testing the Pkg after importing, rather than after freeing it
+    #@eval import Example
+    #@eval import $(Symbol(pkg2))
     @test Pkg.installed()[pkg2] == v"0.1.0"
     Pkg.REPLMode.pkgstr("free $pkg2")
     @test_throws CommandError Pkg.REPLMode.pkgstr("free $pkg2")

stdlib/Pkg/test/test_packages/UnregisteredWithoutProject/src/UnregisteredWithoutProject.jl

@@ -1,6 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__()
 module UnregisteredWithoutProject
 
 if !isfile(joinpath(@__DIR__, "..", "deps", "deps.jl"))

stdlib/Printf/src/Printf.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 module Printf
 # the macro implementations here exactly mirrors the
 # macros left in base/printf.jl, and uses the utility there

stdlib/Profile/src/Profile.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 """
 Profiling support, main entry point is the [`@profile`](@ref) macro.
 """

stdlib/REPL/src/REPL.jl

@@ -1,7 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-__precompile__(true)
-
 module REPL
 
 using Base.Meta, Sockets