try to bring some sanity to the embedding makefile

we'll still want to eventually decide how to make julia-config.jl output correct relative rpaths
but this should at least get `make -C embedding' working correctly

.travis.yml

@@ -96,6 +96,7 @@ before_install:
 script:
     - echo BUILDOPTS=$BUILDOPTS
     - export BUILDOPTS
+    # compile / install dependencies
     - contrib/download_cmake.sh
     - make -C moreutils mispipe
     - make $BUILDOPTS -C base version_git.jl.phony
@@ -113,6 +114,7 @@ script:
           cat deps.log;
           echo "-- end of deps build log -----------------------------------------------";
           false; }
+    # compile / install Julia
     - make $BUILDOPTS NO_GIT=1 prefix=/tmp/julia install | moreutils/ts -s "%.s"
     - make $BUILDOPTS NO_GIT=1 build-stats
     - du -sk /tmp/julia/*
@@ -121,16 +123,24 @@ script:
             install -pm755 usr/lib/lib${name}*.dylib* /tmp/julia/lib/julia/;
         done;
       fi
-    - make $BUILDOPTS NO_GIT=1 prefix=/tmp/julia -C examples
     - cd .. && mv julia julia2
+    # run tests
     - /tmp/julia/bin/julia --precompiled=no -e 'true' &&
         /tmp/julia/bin/julia-debug --precompiled=no -e 'true'
     - /tmp/julia/bin/julia -e 'versioninfo()'
+    - pushd /tmp/julia/share/julia/test
     - export JULIA_CPU_CORES=2 && export JULIA_TEST_MAXRSS_MB=600 &&
-        cd /tmp/julia/share/julia/test &&
         /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
-        /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online download pkg embedding
-    - cd `dirname $TRAVIS_BUILD_DIR` && mv julia2 julia &&
+        /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online download pkg
+    - popd
+    # test that the embedding code works on our installation
+    - mkdir /tmp/embedding-test &&
+        make check -C /tmp/julia/share/doc/julia/examples/embedding \
+             JULIA="DYLD_FALLBACK_LIBRARY_PATH='$DYLD_FALLBACK_LIBRARY_PATH' /tmp/julia/bin/julia" \
+             BIN=/tmp/embedding-test \
+             "$(cd julia2 && make print-CC)"
+    # restore initial state and prepare for travis caching
+    - mv julia2 julia &&
         rm -f julia/deps/scratch/libgit2-*/CMakeFiles/CMakeOutput.log
 # uncomment the following if failures are suspected to be due to the out-of-memory killer
 #    - dmesg

Make.inc

@@ -219,7 +219,6 @@ libdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(libdir))
 build_private_libdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(build_bindir) $(build_private_libdir))
 private_libdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(private_libdir))
 datarootdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(datarootdir))
-libexecdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(libexecdir))
 docdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(docdir))
 sysconfdir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(sysconfdir))
 includedir_rel := $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(includedir))

Makefile

@@ -64,12 +64,12 @@ clean-docdir:
 	@-rm -fr $(abspath $(build_docdir))
 
 $(build_prefix)/.examples: $(wildcard $(JULIAHOME)/examples/*.jl) \
-                           $(shell find $(JULIAHOME)/examples/clustermanager)
+                           $(shell find $(JULIAHOME)/examples/clustermanager) \
+                           $(shell find $(JULIAHOME)/examples/embedding)
 	@echo Copying in usr/share/doc/julia/examples
 	@-rm -fr $(build_docdir)/examples
 	@mkdir -p $(build_docdir)/examples
 	@cp -R $(JULIAHOME)/examples/*.jl $(build_docdir)/examples/
-	@cp -R $(JULIAHOME)/examples/Makefile $(build_docdir)/examples/
 	@cp -R $(JULIAHOME)/examples/clustermanager $(build_docdir)/examples/
 	@cp -R $(JULIAHOME)/examples/embedding $(build_docdir)/examples
 	@echo 1 > $@
@@ -334,12 +334,11 @@ endef
 
 install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index.html
 	@$(MAKE) $(QUIET_MAKE) all
-	@for subdir in $(bindir) $(libexecdir) $(datarootdir)/julia/site/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir); do \
+	@for subdir in $(bindir) $(datarootdir)/julia/site/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir); do \
 		mkdir -p $(DESTDIR)$$subdir; \
 	done
 
 	$(INSTALL_M) $(build_bindir)/julia* $(DESTDIR)$(bindir)/
-	-cp -a $(build_libexecdir) $(DESTDIR)$(prefix)
 ifeq ($(OS),WINNT)
 	-$(INSTALL_M) $(build_bindir)/*.dll $(DESTDIR)$(bindir)/
 	-$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/

appveyor.yml

@@ -56,4 +56,6 @@ test_script:
   - usr\bin\julia -e "versioninfo()"
   - usr\bin\julia --precompiled=no -e "true"
   - cd julia-* && .\bin\julia.exe --check-bounds=yes share\julia\test\runtests.jl all &&
-      .\bin\julia.exe --check-bounds=yes share\julia\test\runtests.jl libgit2-online download pkg embedding
+      .\bin\julia.exe --check-bounds=yes share\julia\test\runtests.jl libgit2-online download pkg
+  - cd ..
+  - usr\bin\julia usr\share\doc\julia\examples\embedding\embedding-test.jl examples\embedding\embedding.exe

base/Makefile

@@ -63,7 +63,6 @@ endif
 	@echo "const DOCDIR = \"$(docdir_rel)\"" >> $@
 	@echo "const LIBDIR = \"$(libdir_rel)\"" >> $@
 	@echo "const PRIVATE_LIBDIR = \"$(private_libdir_rel)\"" >> $@
-	@echo "const LIBEXECDIR = \"$(libexecdir_rel)\"" >> $@
 	@echo "const INCLUDEDIR = \"$(includedir_rel)\"" >> $@
 
 	@# This to ensure that we always rebuild this file, but only when it is modified do we touch build_h.jl,

contrib/julia-config.jl

@@ -1,18 +1,22 @@
 #!/usr/bin/env julia
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-const options =
-[
+const options = [
     "--cflags",
     "--ldflags",
     "--ldlibs"
 ];
 
 threadingOn() = ccall(:jl_threading_enabled, Cint, ()) != 0
 
+function shell_escape(str)
+    str = replace(str, "'", "'\''")
+    return "'$str'"
+end
+
 function imagePath()
     opts = Base.JLOptions()
-    unsafe_string(opts.image_file)
+    return unsafe_string(opts.image_file)
 end
 
 function libDir()
@@ -23,35 +27,14 @@ function libDir()
     end
 end
 
-private_libDir() = joinpath(JULIA_HOME, Base.PRIVATE_LIBDIR)
+private_libDir() = abspath(JULIA_HOME, Base.PRIVATE_LIBDIR)
 
 function includeDir()
-    joinpath(match(r"(.*)(bin)",JULIA_HOME).captures[1],"include","julia")
-end
-
-function unixInitDir()
-    filePart = split(imagePath(),"/")[end]
-    return match(Regex("(.*)(/julia/$filePart)"),imagePath()).captures[1]
-end
-
-function windowsInitDir()
-    if imagePath()[end-1:end] == "ji"
-        return match(r"(.*)(\\julia\\sys.ji)",imagePath()).captures[1]
-    else
-        return match(r"(.*)(\\julia\\sys.dll)",imagePath()).captures[1]
-    end
-end
-
-function initDir()
-    if is_unix()
-        return unixInitDir()
-    else
-        return windowsInitDir()
-    end
+    return abspath(JULIA_HOME, Base.INCLUDEDIR, "julia")
 end
 
 function ldflags()
-    fl = replace("""-L$(libDir())""","\\","\\\\")
+    fl = "-L$(shell_escape(libDir()))"
     if is_windows()
         fl = fl * " -Wl,--stack,8388608"
     elseif is_linux()
@@ -67,27 +50,30 @@ function ldlibs()
         "julia"
     end
     if is_unix()
-        return replace("""-Wl,-rpath,$(libDir()) -Wl,-rpath,$(private_libDir()) -l$libname""","\\","\\\\")
+        return "-Wl,-rpath,$(shell_escape(libDir())) -Wl,-rpath,$(shell_escape(private_libDir())) -l$libname"
     else
         return "-l$libname -lopenlibm"
     end
 end
 
 function cflags()
-    arg1 = replace(initDir(),"\\","\\\\\\\\")
-    arg2 = replace(includeDir(),"\\","\\\\")
-    threading_def = threadingOn() ? "-DJULIA_ENABLE_THREADING=1 " : ""
+    flags = IOBuffer()
+    print(flags, "-std=gnu99")
+    include = shell_escape(includeDir())
+    print(flags, " -I", include)
+    if threadingOn()
+        print(flags, " -DJULIA_ENABLE_THREADING=1")
+    end
     if is_unix()
-        return """-std=gnu99 $(threading_def)-fPIC -DJULIA_INIT_DIR=\\"$arg1\\" -I$arg2"""
-    else
-        return """-std=gnu99 $(threading_def)-DJULIA_INIT_DIR=\\"$arg1\\" -I$arg2"""
+        print(flags, " -fPIC")
     end
+    return String(take!(flags))
 end
 
 function check_args(args)
-    checked = intersect(args,options)
+    checked = intersect(args, options)
     if length(checked) == 0 || length(checked) != length(args)
-        println(STDERR,"Usage: julia-config [",reduce((x,y)->"$x|$y",options),"]")
+        println(STDERR, "Usage: julia-config [", join(options, " | "), "]")
         exit(1)
     end
 end

contrib/windows/msys_build.sh

@@ -204,6 +204,6 @@ echo 'FORCE_ASSERTIONS = 1' >> Make.user
 
 cat Make.user
 make -j3 VERBOSE=1 install
+make VERBOSE=1 -C examples
 cp usr/bin/busybox.exe julia-*/bin
 make build-stats
-make VERBOSE=1 -C examples

examples/Makefile

@@ -1,25 +1,13 @@
+## this is a simple wrapper just to forward on known commands to the
+## embedding example with values pulled from Make.inc
 SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
-BUILDDIR := .
 JULIAHOME := $(abspath $(SRCDIR)/..)
+BUILDDIR := .
 include $(JULIAHOME)/Make.inc
 
-outdir := $(libexecdir)
-
-embedding_binary := $(abspath $(outdir)/embedding$(JULIA_LIBSUFFIX)$(EXE))
-
-release: embedding
-debug: embedding-debug
-
-embedding: $(embedding_binary)
-embedding-debug: $(embedding_binary)
-
-$(embedding_binary): $(wildcard embedding/*)
-	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/examples/embedding $(JULIA_BUILD_MODE) \
-                                JULIA="$(bindir)/julia$(JULIA_LIBSUFFIX)$(EXE)" BIN="$(outdir)" \
-                                SPAWN="$(spawn)" CC="$(CC)"
-
-clean:
-	-rm -f $(embedding_binary) $(embedding_binary)-debug
-
-.PHONY: all embedding clean
-
+release: # default target
+# forward all variables expected by the embedding example
+JULIA:=$(call spawn,$(JULIA_EXECUTABLE))
+BIN:=$(BUILDDIR)/embedding
+CC:=$(CC)
+include $(SRCDIR)/embedding/Makefile

examples/embedding/.gitignore

@@ -0,0 +1,2 @@
+/embedding
+/embedding-debug

examples/embedding/Makefile

@@ -1,3 +1,5 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
 # This Makefile template requires the following variables to be set
 # in the environment or on the command-line:
 #   JULIA: path to julia[.exe] executable
@@ -18,9 +20,10 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 EXE := $(suffix $(abspath $(JULIA)))
 
 # get compiler and linker flags. (see: `contrib/julia-config.jl`)
-CFLAGS += -lm $(shell $(SPAWN) $(JULIA) -e \
-                           'include(joinpath(JULIA_HOME,Base.DATAROOTDIR,"julia","julia-config.jl"))' \
-                           -- --cflags --ldflags --ldlibs)
+JULIA_CONFIG := $(JULIA) -e 'include(joinpath(JULIA_HOME, Base.DATAROOTDIR, "julia", "julia-config.jl"))' --
+CPPFLAGS_ADD :=
+CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags)
+LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs)
 
 DEBUGFLAGS += -g
 
@@ -30,13 +33,21 @@ release: $(BIN)/embedding$(EXE)
 debug:   $(BIN)/embedding-debug$(EXE)
 
 $(BIN)/embedding$(EXE): $(SRCDIR)/embedding.c
-	$(CC) $^ -o $@ $(CFLAGS)
+	$(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS)
 
 $(BIN)/embedding-debug$(EXE): $(SRCDIR)/embedding.c
-	$(CC) $^ -o $@ $(CFLAGS) $(DEBUGFLAGS)
+	$(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS)
+
+check: $(BIN)/embedding$(EXE)
+	$(JULIA) $(SRCDIR)/embedding-test.jl $<
+	@echo SUCCESS
 
 clean:
-	@rm -f $(BIN)/embedding-debug$(EXE) $(BIN)/embedding$(EXE)
+	-rm -f $(BIN)/embedding-debug$(EXE) $(BIN)/embedding$(EXE)
 
-.PHONY: release debug clean
+.PHONY: release debug clean check
 
+# Makefile debugging trick:
+# call print-VARIABLE to see the runtime value of any variable
+print-%:
+	@echo '$*=$($*)'

examples/embedding/embedding-test.jl

@@ -0,0 +1,20 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# tests the output of the embedding example is correct
+using Base.Test
+
+@test length(ARGS) == 1
+let
+    stdout = Pipe()
+    stderr = Pipe()
+    p = spawn(pipeline(Cmd(ARGS), stdin=DevNull, stdout=stdout, stderr=stderr))
+    close(stdout.in)
+    close(stderr.in)
+    stderr_task = @async readlines(stderr)
+    lines = readlines(stdout)
+    @test length(lines) == 6
+    @test parse(Float64, lines[1]) ≈ sqrt(2)
+    lines = wait(stderr_task)
+    @test lines == ["UndefVarError(:this_function_does_not_exist)"]
+    @test success(p)
+end