Merge pull request JuliaLang#13713 from JuliaLang/jcb/ccache

Add option to disable / ignore incremental module precompilation

base/loading.jl

@@ -138,6 +138,9 @@ end
 
 # returns an array of modules loaded, or nothing if failed
 function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteString, toplevel_load::Bool)
+    if JLOptions().use_compilecache == 0
+        return nothing
+    end
     restored = nothing
     if toplevel_load && myid() == 1 && nprocs() > 1
         recompile_stale(mod, path_to_try)
@@ -177,6 +180,9 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteStrin
 end
 
 function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool)
+    if JLOptions().use_compilecache == 0
+        return nothing
+    end
     if node == myid()
         paths = find_all_in_cache_path(mod)
     else
@@ -234,8 +240,10 @@ precompilableerror(ex, c) = false
 # to be prevent it from being precompiled (false).  __precompile__(true) is
 # ignored except within "require" call.
 function __precompile__(isprecompilable::Bool=true)
-    if myid() == 1 && isprecompilable != (0 != ccall(:jl_generating_output, Cint, ())) &&
-        !(isprecompilable && toplevel_load::Bool)
+    if (myid() == 1 &&
+        JLOptions().use_compilecache != 0 &&
+        isprecompilable != (0 != ccall(:jl_generating_output, Cint, ())) &&
+        !(isprecompilable && toplevel_load::Bool))
         throw(PrecompilableError(isprecompilable))
     end
 end
@@ -246,19 +254,21 @@ function require_modname(name::AbstractString)
     # While we could also strip off the absolute path, the user may be
     # deliberately directing to a different file than what got
     # cached. So this takes a conservative approach.
-    if endswith(name, ".jl")
-        tmp = name[1:end-3]
-        for prefix in LOAD_CACHE_PATH
-            path = joinpath(prefix, tmp*".ji")
-            if isfile(path)
-                return tmp
+    if Bool(JLOptions().use_compilecache)
+        if endswith(name, ".jl")
+            tmp = name[1:end-3]
+            for prefix in LOAD_CACHE_PATH
+                path = joinpath(prefix, tmp*".ji")
+                if isfile(path)
+                    return tmp
+                end
             end
         end
     end
-    name
+    return name
 end
 
-doc"""
+"""
     reload(name::AbstractString)
 
 Force reloading of a package, even if it has been loaded before. This is intended for use
@@ -305,7 +315,6 @@ function require(mod::Symbol)
             end
             return
         end
-
         name = string(mod)
         path = find_in_node_path(name, nothing, 1)
         if path === nothing
@@ -347,9 +356,8 @@ include_string(txt::ByteString, fname::ByteString) =
     ccall(:jl_load_file_string, Any, (Ptr{UInt8},Csize_t,Ptr{UInt8},Csize_t),
           txt, sizeof(txt), fname, sizeof(fname))
 
-include_string(txt::AbstractString, fname::AbstractString) = include_string(bytestring(txt), bytestring(fname))
-
-include_string(txt::AbstractString) = include_string(txt, "string")
+include_string(txt::AbstractString, fname::AbstractString="string") =
+    include_string(bytestring(txt), bytestring(fname))
 
 function source_path(default::Union{AbstractString,Void}="")
     t = current_task()
@@ -436,9 +444,7 @@ function create_expr_cache(input::AbstractString, output::AbstractString)
         serialize(io, :(Base._track_dependencies[1] = true))
         serialize(io, :(Base.include($(abspath(input)))))
         if source !== nothing
-            serialize(io, quote
-                      delete!(task_local_storage(), :SOURCE_PATH)
-                      end)
+            serialize(io, :(delete!(task_local_storage(), :SOURCE_PATH)))
         end
         close(io)
         wait(pobj)

base/options.jl

@@ -28,6 +28,7 @@ immutable JLOptions
     worker::Int8
     handle_signals::Int8
     use_precompiled::Int8
+    use_compilecache::Int8
     bindto::Ptr{UInt8}
     outputbc::Ptr{UInt8}
     outputo::Ptr{UInt8}
@@ -36,3 +37,17 @@ immutable JLOptions
 end
 
 JLOptions() = unsafe_load(cglobal(:jl_options, JLOptions))
+
+function show(io::IO, opt::JLOptions)
+    println(io, "JLOptions(")
+    fields = fieldnames(opt)
+    nfields = length(fields)
+    for (i,f) in enumerate(fieldnames(opt))
+        v = getfield(opt,f)
+        if isa(v, Ptr{UInt8})
+            v = v != C_NULL ? bytestring(v) : ""
+        end
+        println(io, "  ", f, " = ", repr(v), i < nfields ? "," : "")
+    end
+    print(io,")")
+end

base/pkg/entry.jl

@@ -543,6 +543,7 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set())
         end
     """
     io, pobj = open(detach(`$(Base.julia_cmd())
+                           --compilecache=$(Bool(Base.JLOptions().use_compilecache) ? "yes" : "no")
                            --history-file=no
                            --color=$(Base.have_color ? "yes" : "no")
                            --eval $code`), "w", STDOUT)

doc/manual/getting-started.rst

@@ -112,6 +112,7 @@ those available for the ``perl`` and ``ruby`` programs::
 
      -J, --sysimage <file>     Start up with the given system image file
      --precompiled={yes|no}    Use precompiled code from system image if available
+     --compilecache={yes|no}   Enable/disable incremental precompilation of modules\n"
      -H, --home <dir>          Set location of julia executable
      --startup-file={yes|no}   Load ~/.juliarc.jl
      -f, --no-startup          Don't load ~/.juliarc (deprecated, use --startup-file=no)

doc/manual/modules.rst

@@ -419,3 +419,9 @@ A few other points to be aware of:
 
 4. WeakRef objects and finalizers are not currently handled properly by the serializer
    (this will be fixed in an upcoming release).
+
+It is sometimes helpful during module development to turn off incremental precompilation.
+The command line flag ``--compilecache={yes|no}`` enables you to toggle module precompilation on and off.
+When Julia is started with ``--compilecache=no`` the serialized modules in the compile cache are ignored when loading modules and module dependencies.
+``Base.compilecache()`` can still be called manually and it will respect ``__precompile__()`` directives for the module.
+The state of this command line flag is passed to ``Pkg.build()`` to disable automatic precompilation triggering when installing, updating, and explicitly building packages.

src/init.c

@@ -76,6 +76,7 @@ jl_options_t jl_options = { 0,    // quiet
 #else
                             JL_OPTIONS_USE_PRECOMPILED_YES,
 #endif
+                            JL_OPTIONS_USE_COMPILECACHE_YES,
                             NULL, // bindto
                             NULL, // outputbc
                             NULL, // outputo

src/julia.h

@@ -1572,6 +1572,7 @@ typedef struct {
     int8_t worker;
     int8_t handle_signals;
     int8_t use_precompiled;
+    int8_t use_compilecache;
     const char *bindto;
     const char *outputbc;
     const char *outputo;
@@ -1621,6 +1622,9 @@ DLLEXPORT int jl_generating_output(void);
 #define JL_OPTIONS_USE_PRECOMPILED_YES 1
 #define JL_OPTIONS_USE_PRECOMPILED_NO 0
 
+#define JL_OPTIONS_USE_COMPILECACHE_YES 1
+#define JL_OPTIONS_USE_COMPILECACHE_NO 0
+
 // Version information
 #include <julia_version.h>
 

test/cmdlineargs.jl

@@ -232,4 +232,10 @@ let exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes`
     @test readchomp(pipeline(ignorestatus(`$exename -f -p`),stderr=`cat`)) == "ERROR: option `-p/--procs` is missing an argument"
     @test readchomp(pipeline(ignorestatus(`$exename -f --inline`),stderr=`cat`)) == "ERROR: option `--inline` is missing an argument"
     @test readchomp(pipeline(ignorestatus(`$exename -f -e "@show ARGS" -now -- julia RUN.jl`),stderr=`cat`)) == "ERROR: unknown option `-n`"
+
+    # --compilecache={yes|no}
+    @test readchomp(`$exename -E "Bool(Base.JLOptions().use_compilecache)"`) == "true"
+    @test readchomp(`$exename --compilecache=yes -E "Bool(Base.JLOptions().use_compilecache)"`) == "true"
+    @test readchomp(`$exename --compilecache=no -E "Bool(Base.JLOptions().use_compilecache)"`) == "false"
+    @test !success(`$exename --compilecache=foo -e "exit(0)"`)
 end

test/compile.jl

@@ -81,8 +81,55 @@ try
     end
     println(STDERR, "\nNOTE: The following 'LoadError: break me' indicates normal operation")
     @test_throws ErrorException Base.require(:FooBar)
+
 finally
     splice!(Base.LOAD_CACHE_PATH, 1)
     splice!(LOAD_PATH, 1)
     rm(dir, recursive=true)
 end
+
+# test --compilecache=no command line option
+dir = mktempdir()
+let dir = mktempdir(),
+    Time_module = :Time4b3a94a1a081a8cb
+
+    try
+        open(joinpath(dir, "$Time_module.jl"), "w") do io
+            write(io, """
+            module $Time_module
+                __precompile__(true)
+                time = Base.time()
+            end
+            """)
+        end
+
+        eval(quote
+            insert!(LOAD_PATH, 1, $(dir))
+            insert!(Base.LOAD_CACHE_PATH, 1, $(dir))
+            Base.compilecache(:Time4b3a94a1a081a8cb)
+        end)
+
+        exename = `$(joinpath(JULIA_HOME, Base.julia_exename())) --precompiled=yes`
+
+        testcode = """
+            insert!(LOAD_PATH, 1, $(repr(dir)))
+            insert!(Base.LOAD_CACHE_PATH, 1, $(repr(dir)))
+            using $Time_module
+            getfield($Time_module, :time)
+        """
+
+        t1_yes = readchomp(`$exename --compilecache=yes -E $(testcode)`)
+        t2_yes = readchomp(`$exename --compilecache=yes -E $(testcode)`)
+        @test t1_yes == t2_yes
+
+        t1_no = readchomp(`$exename --compilecache=no -E $(testcode)`)
+        t2_no = readchomp(`$exename --compilecache=no -E $(testcode)`)
+        @test t1_no != t2_no
+        @test parse(Float64, t1_no) < parse(Float64, t2_no)
+
+    finally
+        splice!(Base.LOAD_CACHE_PATH, 1)
+        splice!(LOAD_PATH, 1)
+        rm(dir, recursive=true)
+    end
+end

ui/repl.c

@@ -47,6 +47,7 @@ static const char opts[]  =
     // startup options
     " -J, --sysimage <file>     Start up with the given system image file\n"
     " --precompiled={yes|no}    Use precompiled code from system image if available\n"
+    " --compilecache={yes|no}   Enable/disable incremental precompilation of modules\n"
     " -H, --home <dir>          Set location of julia executable\n"
     " --startup-file={yes|no}   Load ~/.juliarc.jl\n"
     " -f, --no-startup          Don't load ~/.juliarc (deprecated, use --startup-file=no)\n"
@@ -115,6 +116,7 @@ void parse_opts(int *argcp, char ***argvp)
            opt_output_o,
            opt_output_ji,
            opt_use_precompiled,
+           opt_use_compilecache,
            opt_incremental
     };
     static char* shortopts = "+vhqFfH:e:E:P:L:J:C:ip:O";
@@ -132,6 +134,7 @@ void parse_opts(int *argcp, char ***argvp)
         { "load",            required_argument, 0, 'L' },
         { "sysimage",        required_argument, 0, 'J' },
         { "precompiled",     required_argument, 0, opt_use_precompiled },
+        { "compilecache",    required_argument, 0, opt_use_compilecache },
         { "cpu-target",      required_argument, 0, 'C' },
         { "procs",           required_argument, 0, 'p' },
         { "machinefile",     required_argument, 0, opt_machinefile },
@@ -231,6 +234,14 @@ void parse_opts(int *argcp, char ***argvp)
             else
                 jl_errorf("julia: invalid argument to --precompiled={yes|no} (%s)", optarg);
             break;
+        case opt_use_compilecache:
+            if (!strcmp(optarg,"yes"))
+                jl_options.use_compilecache = JL_OPTIONS_USE_COMPILECACHE_YES;
+            else if (!strcmp(optarg,"no"))
+                jl_options.use_compilecache = JL_OPTIONS_USE_COMPILECACHE_NO;
+            else
+                jl_errorf("julia: invalid argument to --compilecache={yes|no} (%s)", optarg);
+            break;
         case 'C': // cpu-target
             jl_options.cpu_target = strdup(optarg);
             break;