reorganize Base with default packages, installed in site/

as a demo, moves DataFmt out of Base. adds a line to default juliarc so the functions are still available by default.
lilinj2000 · Sep 29, 2017 · 4a6e64d · 4a6e64d
1 parent 7f14cd6
commit 4a6e64d
Show file tree

Hide file tree

Showing 22 changed files with 116 additions and 72 deletions.
diff --git a/Makefile b/Makefile
@@ -19,7 +19,7 @@ default: $(JULIA_BUILD_MODE) # contains either "debug" or "release"
 all: debug release
 
 # sort is used to remove potential duplicates
-DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_man1dir))
+DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/site $(build_man1dir))
 ifneq ($(BUILDROOT),$(JULIAHOME))
 BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src ui doc deps test test/perf examples examples/embedding)
 BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS))
@@ -50,6 +50,11 @@ endif
 
 $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir))))
 $(foreach link,base test,$(eval $(call symlink_target,$(link),$(build_datarootdir)/julia)))
+$(eval $(call symlink_target,stdlib,$(build_datarootdir)/julia/site))
+
+build_defaultpkgdir = $(build_datarootdir)/julia/site/$(shell echo $(VERSDIR))
+$(build_defaultpkgdir): $(build_datarootdir)/julia/site/stdlib
+	@mv $(build_datarootdir)/julia/site/stdlib $@
 
 julia_flisp.boot.inc.phony: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src julia_flisp.boot.inc.phony
@@ -81,7 +86,7 @@ ifndef JULIA_VAGRANT_BUILD
 endif
 endif
 
-julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test
+julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia/test $(build_defaultpkgdir)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/deps
 
 julia-base: julia-deps $(build_sysconfdir)/julia/juliarc.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl

diff --git a/README.md b/README.md
@@ -372,11 +372,12 @@ It is highly recommended to start with a fresh clone of the Julia repository.
 
 The Julia source code is organized as follows:
 
-    base/          source code for Julia's standard library
+    base/          source code for the Base module (part of Julia's standard library)
+    stdlib/        source code for other standard library packages
     contrib/       editor support for Julia source, miscellaneous scripts
     deps/          external dependencies
-    doc/manual     source for the user manual
-    doc/stdlib     source for standard library function help text
+    doc/src/manual source for the user manual
+    doc/src/stdlib source for standard library function reference
     examples/      example Julia programs
     src/           source for Julia language core
     test/          test suites

diff --git a/base/deprecated.jl b/base/deprecated.jl
@@ -1724,19 +1724,12 @@ import .LinAlg: diagm
 @deprecate_binding φ          MathConstants.φ
 @deprecate_binding golden     MathConstants.golden
 
-# deprecate writecsv
-@deprecate writecsv(io, a; opts...) writedlm(io, a, ','; opts...)
-
 # PR #23271
 function IOContext(io::IO; kws...)
     depwarn("IOContext(io, k=v, ...) is deprecated, use IOContext(io, :k => v, ...) instead.", :IOContext)
     IOContext(io, (k=>v for (k, v) in kws)...)
 end
 
-# deprecate readcsv
-@deprecate readcsv(io; opts...) readdlm(io, ','; opts...)
-@deprecate readcsv(io, T::Type; opts...) readdlm(io, ',', T; opts...)
-
 @deprecate IOContext(io::IO, key, value) IOContext(io, key=>value)
 
 # PR #23485

diff --git a/base/exports.jl b/base/exports.jl
@@ -1048,7 +1048,6 @@ export
     readbytes!,
     readchomp,
     readdir,
-    readdlm,
     readline,
     readlines,
     readuntil,
@@ -1070,7 +1069,6 @@ export
     unmark,
     watch_file,
     write,
-    writedlm,
     TCPSocket,
     UDPSocket,
 

diff --git a/base/initdefs.jl b/base/initdefs.jl
@@ -69,7 +69,7 @@ functionality is experimental and may break or change in Julia 1.0.
 const LOAD_PATH = Any[]
 const LOAD_CACHE_PATH = String[]
 
-function init_load_path()
+function init_load_path(JULIA_HOME = JULIA_HOME)
     vers = "v$(VERSION.major).$(VERSION.minor)"
     if haskey(ENV, "JULIA_LOAD_PATH")
         prepend!(LOAD_PATH, split(ENV["JULIA_LOAD_PATH"], @static Sys.iswindows() ? ';' : ':'))

diff --git a/base/io.jl b/base/io.jl
@@ -779,3 +779,26 @@ function skipchars(io::IO, pred; linecomment=nothing)
     end
     return io
 end
+
+"""
+    countlines(io::IO, eol::Char='\\n')
+
+Read `io` until the end of the stream/file and count the number of lines. To specify a file
+pass the filename as the first argument. EOL markers other than `'\\n'` are supported by
+passing them as the second argument.
+"""
+function countlines(io::IO, eol::Char='\n')
+    isascii(eol) || throw(ArgumentError("only ASCII line terminators are supported"))
+    aeol = UInt8(eol)
+    a = Vector{UInt8}(8192)
+    nl = 0
+    while !eof(io)
+        nb = readbytes!(io, a)
+        @simd for i=1:nb
+            @inbounds nl += a[i] == aeol
+        end
+    end
+    nl
+end
+
+countlines(f::AbstractString, eol::Char='\n') = open(io->countlines(io,eol), f)::Int
diff --git a/base/sysimg.jl b/base/sysimg.jl
@@ -360,8 +360,6 @@ include("mmap.jl")
 import .Mmap
 
 # utilities - timing, help, edit
-include("datafmt.jl")
-using .DataFmt
 include("deepcopy.jl")
 include("interactiveutil.jl")
 include("summarysize.jl")
@@ -453,4 +451,12 @@ end # baremodule Base
 
 using Base
 
+# set up load path to be able to find stdlib packages
+Base.init_load_path(ccall(:jl_get_julia_home, Any, ()))
+
+# load some stdlib packages but don't put their names in Main
+Base.require(:DelimitedFiles)
+
+empty!(LOAD_PATH)
+
 Base.isfile("userimg.jl") && Base.include(Main, "userimg.jl")
diff --git a/doc/make.jl b/doc/make.jl
@@ -16,6 +16,13 @@ end
 
 # Documenter Setup.
 
+# make links for stdlib package docs
+if Sys.iswindows()
+    cp("../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
+else
+    symlink("../../../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md")
+end
+
 const PAGES = [
     "Home" => "index.md",
     "Manual" => [
@@ -69,6 +76,7 @@ const PAGES = [
         "stdlib/linalg.md",
         "stdlib/constants.md",
         "stdlib/file.md",
+        "stdlib/delimitedfiles.md",
         "stdlib/io-network.md",
         "stdlib/punctuation.md",
         "stdlib/sort.md",
@@ -116,9 +124,11 @@ const PAGES = [
     ],
 ]
 
+using DelimitedFiles
+
 makedocs(
     build     = joinpath(pwd(), "_build/html/en"),
-    modules   = [Base, Core, BuildSysImg],
+    modules   = [Base, Core, BuildSysImg, DelimitedFiles],
     clean     = false,
     doctest   = "doctest" in ARGS,
     linkcheck = "linkcheck" in ARGS,

diff --git a/doc/src/stdlib/.gitignore b/doc/src/stdlib/.gitignore
@@ -0,0 +1 @@
+delimitedfiles.md
diff --git a/doc/src/stdlib/index.md b/doc/src/stdlib/index.md
@@ -10,6 +10,7 @@
   * [Linear Algebra](@ref)
   * [Constants](@ref lib-constants)
   * [Filesystem](@ref)
+  * [Delimited Files](@ref)
   * [I/O and Network](@ref)
   * [Punctuation](@ref)
   * [Sorting and Related Functions](@ref)

diff --git a/doc/src/stdlib/io-network.md b/doc/src/stdlib/io-network.md
@@ -47,7 +47,7 @@ Base.redirect_stdin(::Function, ::Any)
 Base.readchomp
 Base.truncate
 Base.skipchars
-Base.DataFmt.countlines
+Base.countlines
 Base.PipeBuffer
 Base.readavailable
 Base.IOContext
@@ -77,13 +77,6 @@ Base.readline
 Base.readuntil
 Base.readlines
 Base.eachline
-Base.DataFmt.readdlm(::Any, ::Char, ::Type, ::Char)
-Base.DataFmt.readdlm(::Any, ::Char, ::Char)
-Base.DataFmt.readdlm(::Any, ::Char, ::Type)
-Base.DataFmt.readdlm(::Any, ::Char)
-Base.DataFmt.readdlm(::Any, ::Type)
-Base.DataFmt.readdlm(::Any)
-Base.DataFmt.writedlm
 Base.Base64.Base64EncodePipe
 Base.Base64.Base64DecodePipe
 Base.Base64.base64encode

diff --git a/etc/juliarc.jl b/etc/juliarc.jl
@@ -1,3 +1,4 @@
 # This file should contain site-specific commands to be executed on Julia startup
 # Users may store their own personal commands in the user home directory `homedir()`, in a file named `.juliarc.jl`
 
+using DelimitedFiles
diff --git a/stdlib/DelimitedFiles/docs/src/index.md b/stdlib/DelimitedFiles/docs/src/index.md
@@ -0,0 +1,11 @@
+# Delimited Files
+
+```@docs
+DelimitedFiles.readdlm(::Any, ::Char, ::Type, ::Char)
+DelimitedFiles.readdlm(::Any, ::Char, ::Char)
+DelimitedFiles.readdlm(::Any, ::Char, ::Type)
+DelimitedFiles.readdlm(::Any, ::Char)
+DelimitedFiles.readdlm(::Any, ::Type)
+DelimitedFiles.readdlm(::Any)
+DelimitedFiles.writedlm
+```
diff --git a/base/datafmt.jl → stdlib/DelimitedFiles/src/DelimitedFiles.jl b/base/datafmt.jl → stdlib/DelimitedFiles/src/DelimitedFiles.jl
@@ -1,12 +1,16 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-## file formats ##
+__precompile__(true)
 
-module DataFmt
+module DelimitedFiles
 
 import Base: _default_delims, tryparse_internal, show
 
-export countlines, readdlm, writedlm
+export readdlm, writedlm
+
+Base.@deprecate readcsv(io; opts...) readdlm(io, ','; opts...)
+Base.@deprecate readcsv(io, T::Type; opts...) readdlm(io, ',', T; opts...)
+Base.@deprecate writecsv(io, a; opts...) writedlm(io, a, ','; opts...)
 
 invalid_dlm(::Type{Char})   = reinterpret(Char, 0xfffffffe)
 invalid_dlm(::Type{UInt8})  = 0xfe
@@ -15,29 +19,6 @@ invalid_dlm(::Type{UInt32}) = 0xfffffffe
 
 const offs_chunk_size = 5000
 
-countlines(f::AbstractString, eol::Char='\n') = open(io->countlines(io,eol), f)::Int
-
-"""
-    countlines(io::IO, eol::Char='\\n')
-
-Read `io` until the end of the stream/file and count the number of lines. To specify a file
-pass the filename as the first argument. EOL markers other than `'\\n'` are supported by
-passing them as the second argument.
-"""
-function countlines(io::IO, eol::Char='\n')
-    isascii(eol) || throw(ArgumentError("only ASCII line terminators are supported"))
-    aeol = UInt8(eol)
-    a = Vector{UInt8}(8192)
-    nl = 0
-    while !eof(io)
-        nb = readbytes!(io, a)
-        @simd for i=1:nb
-            @inbounds nl += a[i] == aeol
-        end
-    end
-    nl
-end
-
 """
     readdlm(source, T::Type; options...)
 
@@ -698,4 +679,4 @@ writedlm(io, a; opts...) = writedlm(io, a, '\t'; opts...)
 show(io::IO, ::MIME"text/csv", a) = writedlm(io, a, ',')
 show(io::IO, ::MIME"text/tab-separated-values", a) = writedlm(io, a, '\t')
 
-end # module DataFmt
+end # module DelimitedFiles
diff --git a/test/datafmt.jl → stdlib/DelimitedFiles/test/runtests.jl b/test/datafmt.jl → stdlib/DelimitedFiles/test/runtests.jl
@@ -1,18 +1,7 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-@testset "countlines" begin
-    @test countlines(IOBuffer("\n")) == 1
-    @test countlines(IOBuffer("\n"),'\r') == 0
-    @test countlines(IOBuffer("\n\n\n\n\n\n\n\n\n\n")) == 10
-    @test countlines(IOBuffer("\n \n \n \n \n \n \n \n \n \n")) == 10
-    @test countlines(IOBuffer("\r\n \r\n \r\n \r\n \r\n")) == 5
-    file = tempname()
-    write(file,"Spiffy header\nspectacular first row\neven better 2nd row\nalmost done\n")
-    @test countlines(file) == 4
-    @test countlines(file,'\r') == 0
-    @test countlines(file,'\n') == 4
-    rm(file)
-end
+using Base.Test
+using DelimitedFiles
 
 isequaldlm(m1, m2, t) = isequal(m1, m2) && (eltype(m1) == eltype(m2) == t)
 
@@ -295,3 +284,9 @@ end
 
 # issue #11484: useful error message for invalid readdlm filepath arguments
 @test_throws ArgumentError readdlm(tempdir())
+
+# displaying as text/csv
+let d = TextDisplay(IOBuffer())
+    display(d, "text/csv", [3 1 4])
+    @test String(take!(d.io)) == "3,1,4\n"
+end
diff --git a/test/choosetests.jl b/test/choosetests.jl
@@ -32,7 +32,7 @@ function choosetests(choices = [])
         "bigfloat", "sorting", "statistics", "spawn", "backtrace",
         "file", "read", "mmap", "version", "resolve",
         "pollfd", "mpfr", "broadcast", "complex", "socket",
-        "floatapprox", "datafmt", "reflection", "regex", "float16",
+        "floatapprox", "stdlib", "reflection", "regex", "float16",
         "combinatorics", "sysinfo", "env", "rounding", "ranges", "mod2pi",
         "euler", "show", "lineedit", "replcompletions", "repl",
         "replutil", "sets", "test", "goto", "llvmcall", "llvmcall2", "grisu",

diff --git a/test/compile.jl b/test/compile.jl
@@ -207,8 +207,11 @@ try
         @test map(x -> x[1],  sort(deps)) == [Foo_file, joinpath(dir, "bar.jl"), joinpath(dir, "foo.jl")]
 
         modules, deps1 = Base.cache_dependencies(cachefile)
-        @test modules == Dict(s => Base.module_uuid(getfield(Foo, s)) for s in
-                                    [:Base, :Core, Foo2_module, FooBase_module, :Main])
+        @test modules == merge(Dict(s => Base.module_uuid(getfield(Foo, s)) for s in
+                                    [:Base, :Core, Foo2_module, FooBase_module, :Main]),
+                               # plus modules included in the system image
+                               Dict(s => Base.module_uuid(Base.root_module(s)) for s in
+                                    [:DelimitedFiles]))
         @test deps == deps1
 
         @test current_task()(0x01, 0x4000, 0x30031234) == 2

diff --git a/test/offsetarray.jl b/test/offsetarray.jl
@@ -2,6 +2,7 @@
 
 isdefined(Main, :TestHelpers) || @eval Main include(joinpath(dirname(@__FILE__), "TestHelpers.jl"))
 using Main.TestHelpers.OAs
+using DelimitedFiles
 
 const OAs_name = join(fullname(OAs), ".")
 

diff --git a/test/read.jl b/test/read.jl
@@ -1,5 +1,7 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
+using DelimitedFiles
+
 mktempdir() do dir
 
 tasks = []
@@ -521,3 +523,17 @@ close(f2)
 rm(f)
 
 end # mktempdir() do dir
+
+@testset "countlines" begin
+    @test countlines(IOBuffer("\n")) == 1
+    @test countlines(IOBuffer("\n"),'\r') == 0
+    @test countlines(IOBuffer("\n\n\n\n\n\n\n\n\n\n")) == 10
+    @test countlines(IOBuffer("\n \n \n \n \n \n \n \n \n \n")) == 10
+    @test countlines(IOBuffer("\r\n \r\n \r\n \r\n \r\n")) == 5
+    file = tempname()
+    write(file,"Spiffy header\nspectacular first row\neven better 2nd row\nalmost done\n")
+    @test countlines(file) == 4
+    @test countlines(file,'\r') == 0
+    @test countlines(file,'\n') == 4
+    rm(file)
+end
diff --git a/test/show.jl b/test/show.jl
@@ -786,10 +786,7 @@ end
 # don't use julia-specific `f` in Float32 printing (PR #18053)
 @test sprint(print, 1f-7) == "1.0e-7"
 
-# test that the REPL TextDisplay works for displaying arbitrary textual MIME types
 let d = TextDisplay(IOBuffer())
-    display(d, "text/csv", [3 1 4])
-    @test String(take!(d.io)) == "3,1,4\n"
     @test_throws MethodError display(d, "text/foobar", [3 1 4])
     try
         display(d, "text/foobar", [3 1 4])

diff --git a/test/sparse/cholmod.jl b/test/sparse/cholmod.jl
@@ -3,6 +3,7 @@
 srand(123)
 
 using Base.SparseArrays.CHOLMOD
+using DelimitedFiles
 
 # based on deps/SuiteSparse-4.0.2/CHOLMOD/Demo/
 

diff --git a/test/stdlib.jl b/test/stdlib.jl
@@ -0,0 +1,7 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# test default packages
+
+cd(joinpath(JULIA_HOME,"..","share","julia","site","v$(VERSION.major).$(VERSION.minor)")) do
+    Pkg.Entry.test(convert(Vector{AbstractString}, readdir()))
+end