Skip to content

Commit

Permalink
cleanup dependencies and rely on logstash-core gem
Browse files Browse the repository at this point in the history
remove unused load path

use either local core lib dir or logstash-core gem

include spec in logstash-core

do not include lib, spec and locales dirs in package root

environment bootstrapping in lib/bootstrap.rb

added comments

move pluginmanager out of logstash-core

kafla installation solved by pluginmanager refactor

refactor bootstrap code

refactor shell scripts to simplify and DRY, indirect rspec command to our ruby launcher

add bin/rspec

cut by half the bin/plugin and bin/rspec startup time

fix drip context

use printf instead of tr

updated Windows bin scripts

missing gemspec in gemspec

use gem instead of gemspec so our plugin manager can correctly install logstash-core

generate packages including locally built logstash-core gem

move jenkins developmnent dependencies into Gemfile, they do not belong in logstash-core

path leftover

clean help agent help usage message and remore rspec in available command

comments cosmetics

update Bundler dependency, all recent testing have been made with 1.9.4

updated .lock file with regard to updated Gemfile

cleanup gemspec, Gemfile and regen .lock file

added progress output

avoid dual require on version

closes elastic#3096
  • Loading branch information
colinsurprenant committed May 4, 2015
1 parent 8300255 commit 48bcc5b
Show file tree
Hide file tree
Showing 44 changed files with 426 additions and 526 deletions.
7 changes: 6 additions & 1 deletion Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,9 @@
# If you modify this file manually all comments and formatting will be lost.

source "https://rubygems.org"
gemspec :name => "logstash-core"
gem "logstash-core", "2.0.0.dev", :path => "."
gem "file-dependencies", "0.1.6"
gem "ci_reporter", "1.9.3", :group => :development
gem "simplecov", :group => :development
gem "coveralls", :group => :development
gem "rspec", "~> 2.14.0", :group => :development
113 changes: 31 additions & 82 deletions Gemfile.jruby-1.9.lock
Original file line number Diff line number Diff line change
@@ -1,94 +1,57 @@
PATH
remote: .
specs:
logstash (2.0.0.dev-java)
cabin (>= 0.7.0)
clamp
file-dependencies (= 0.1.6)
filesize
ftw (~> 0.0.40)
logstash-core (2.0.0.dev-java)
cabin (~> 0.7.0)
clamp (~> 0.6.5)
filesize (= 0.0.4)
i18n (= 0.6.9)
jar-dependencies (= 0.1.7)
jrjackson
maven-tools (= 1.0.7)
mime-types
minitar
pry
rack
ruby-maven (= 3.1.1.0.8)
sinatra
stud
treetop (~> 1.4.0)
jrjackson (~> 0.2.8)
minitar (~> 0.5.4)
pry (~> 0.10.1)
stud (~> 0.0.19)
treetop (< 1.5.0)

GEM
remote: https://rubygems.org/
specs:
addressable (2.3.7)
axiom-types (0.1.1)
descendants_tracker (~> 0.0.4)
ice_nine (~> 0.11.0)
thread_safe (~> 0.3, >= 0.3.1)
backports (3.6.4)
builder (3.2.2)
cabin (0.7.1)
ci_reporter (1.9.3)
builder (>= 2.1.2)
clamp (0.6.3)
clamp (0.6.5)
coderay (1.1.0)
coercible (1.0.0)
descendants_tracker (~> 0.0.1)
coveralls (0.7.10)
multi_json (~> 1.10)
coveralls (0.8.1)
json (~> 1.8)
rest-client (>= 1.6.8, < 2)
simplecov (~> 0.9.1)
simplecov (~> 0.10.0)
term-ansicolor (~> 1.3)
thor (~> 0.19.1)
descendants_tracker (0.0.4)
thread_safe (~> 0.3, >= 0.3.1)
diff-lcs (1.2.5)
docile (1.1.5)
equalizer (0.0.9)
ffi (1.9.6-java)
domain_name (0.5.24)
unf (>= 0.0.5, < 1.0.0)
ffi (1.9.8-java)
file-dependencies (0.1.6)
minitar
filesize (0.0.4)
ftw (0.0.42)
addressable
backports (>= 2.6.2)
cabin (> 0)
http_parser.rb (~> 0.6)
gem_publisher (1.5.0)
http_parser.rb (0.6.0-java)
http-cookie (1.0.2)
domain_name (~> 0.5)
i18n (0.6.9)
ice_nine (0.11.1)
insist (1.0.0)
jar-dependencies (0.1.7)
jrjackson (0.2.8)
logstash-devutils (0.0.12-java)
gem_publisher
insist (= 1.0.0)
jar-dependencies
minitar
rake
rspec (~> 2.14.0)
maven-tools (1.0.7)
virtus (~> 1.0)
json (1.8.2-java)
method_source (0.8.2)
mime-types (2.4.3)
mime-types (2.5)
minitar (0.5.4)
multi_json (1.10.1)
netrc (0.10.2)
netrc (0.10.3)
polyglot (0.3.5)
pry (0.10.1-java)
coderay (~> 1.1.0)
method_source (~> 0.8.1)
slop (~> 3.4)
spoon (~> 0.0)
rack (1.6.0)
rack-protection (1.5.3)
rack
rake (10.4.2)
rest-client (1.7.3)
rest-client (1.8.0)
http-cookie (>= 1.0.2, < 2.0)
mime-types (>= 1.16, < 3.0)
netrc (~> 0.7)
rspec (2.14.1)
Expand All @@ -99,45 +62,31 @@ GEM
rspec-expectations (2.14.5)
diff-lcs (>= 1.1.3, < 2.0)
rspec-mocks (2.14.6)
ruby-maven (3.1.1.0.8)
maven-tools (~> 1.0.1)
ruby-maven-libs (= 3.1.1)
ruby-maven-libs (3.1.1)
simplecov (0.9.2)
simplecov (0.10.0)
docile (~> 1.1.0)
multi_json (~> 1.0)
simplecov-html (~> 0.9.0)
simplecov-html (0.9.0)
sinatra (1.4.5)
rack (~> 1.4)
rack-protection (~> 1.4)
tilt (~> 1.3, >= 1.3.4)
json (~> 1.8)
simplecov-html (~> 0.10.0)
simplecov-html (0.10.0)
slop (3.6.0)
spoon (0.0.4)
ffi
stud (0.0.19)
term-ansicolor (1.3.0)
tins (~> 1.0)
thor (0.19.1)
thread_safe (0.3.4-java)
tilt (1.4.1)
tins (1.3.4)
tins (1.5.1)
treetop (1.4.15)
polyglot
polyglot (>= 0.3.1)
virtus (1.0.4)
axiom-types (~> 0.1)
coercible (~> 1.0)
descendants_tracker (~> 0.0, >= 0.0.3)
equalizer (~> 0.0, >= 0.0.9)
unf (0.1.4-java)

PLATFORMS
java

DEPENDENCIES
ci_reporter (= 1.9.3)
coveralls
logstash!
logstash-devutils
file-dependencies (= 0.1.6)
logstash-core (= 2.0.0.dev)!
rspec (~> 2.14.0)
simplecov
6 changes: 2 additions & 4 deletions bin/bundle
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,14 @@
# Exit cleanly from an early interrupt
Signal.trap("INT") { exit 1 }

$LOAD_PATH.unshift(File.expand_path(File.join("__FILE__", "..", "lib")))

require "logstash/environment"
require_relative "../lib/bootstrap/environment"
Gem.clear_paths
Gem.paths = ENV['GEM_HOME'] = ENV['GEM_PATH'] = LogStash::Environment.logstash_gem_home

require "bundler"
require "bundler/cli"
require "bundler/friendly_errors"
require "logstash/patches/bundler"
LogStash::Bundler.patch!

Bundler.with_friendly_errors do
Bundler::CLI.start(ARGV, :debug => true)
Expand Down
33 changes: 13 additions & 20 deletions bin/logstash
Original file line number Diff line number Diff line change
Expand Up @@ -4,35 +4,28 @@
# This is most useful when done from a git checkout.
#
# Usage:
# bin/logstash <command> [arguments]
# bin/logstash <command> [arguments]
#
# See 'bin/logstash help' for a list of commands.
#
# Defaults you can override with environment variables
LS_HEAP_SIZE="${LS_HEAP_SIZE:=500m}"

unset CDPATH
basedir=$(cd `dirname $0`/..; pwd)
. "${basedir}/bin/logstash.lib.sh"
# Supported environment variables:
# LS_HEAP_SIZE="xxx" size for the -Xmx${LS_HEAP_SIZE} maximum Java heap size option, default is "500m"
# LS_JAVA_OPTS="xxx" to append extra options to the defaults JAVA_OPTS provided by logstash
# JAVA_OPTS="xxx" to *completely override* the defauls set of JAVA_OPTS provided by logstash
#
# Development environment variables:
# USE_RUBY=1 to force use the local "ruby" command to launch logstash instead of using the vendored JRuby
# USE_DRIP=1 to force use drip
# DEBUG=1 to output debugging information

. "$(cd `dirname $0`/..; pwd)/bin/logstash.lib.sh"
setup

# Export these so that they can be picked up by file input (and others?).
export HOME SINCEDB_DIR

case $1 in
-*)
if [ -z "$VENDORED_JRUBY" ] ; then
exec "${RUBYCMD}" "${basedir}/lib/logstash/runner.rb" "agent" "$@"
else
exec "$JRUBY_BIN" $(jruby_opts) "${basedir}/lib/logstash/runner.rb" "agent" "$@"
fi
ruby_exec "${LOGSTASH_HOME}/lib/bootstrap/environment.rb" "logstash/runner.rb" "agent" "$@"
;;
*)
if [ -z "$VENDORED_JRUBY" ] ; then
exec "${RUBYCMD}" "${basedir}/lib/logstash/runner.rb" "$@"
else
exec "$JRUBY_BIN" $(jruby_opts) "${basedir}/lib/logstash/runner.rb" "$@"
fi
ruby_exec "${LOGSTASH_HOME}/lib/bootstrap/environment.rb" "logstash/runner.rb" "$@"
;;
esac
4 changes: 0 additions & 4 deletions bin/logstash-test

This file was deleted.

8 changes: 4 additions & 4 deletions bin/logstash.bat
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ set first_arg=%1
setlocal EnableDelayedExpansion
if "!first_arg:~0,1!" equ "-" (
if "%VENDORED_JRUBY%" == "" (
%RUBYCMD% "%LS_HOME%\lib\logstash\runner.rb" agent %*
%RUBYCMD% "%LS_HOME%\lib\bootstrap\environment.rb" "logstash\runner.rb" agent %*
) else (
%JRUBY_BIN% %jruby_opts% "%LS_HOME%\lib\logstash\runner.rb" agent %*
%JRUBY_BIN% %jruby_opts% "%LS_HOME%\lib\bootstrap\environment.rb" "logstash\runner.rb" agent %*
)
) else (
if "%VENDORED_JRUBY%" == "" (
%RUBYCMD% "%LS_HOME%\lib\logstash\runner.rb" %*
%RUBYCMD% "%LS_HOME%\lib\bootstrap\environment.rb" "logstash\runner.rb" %*
) else (
%JRUBY_BIN% %jruby_opts% "%LS_HOME%\lib\logstash\runner.rb" %*
%JRUBY_BIN% %jruby_opts% "%LS_HOME%\lib\bootstrap\environment.rb" "logstash\runner.rb" %*
)
)

Expand Down
38 changes: 30 additions & 8 deletions bin/logstash.lib.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
basedir=$(cd `dirname $0`/..; pwd)
unset CDPATH
LOGSTASH_HOME=$(cd `dirname $0`/..; pwd)
export LOGSTASH_HOME

# Defaults you can override with environment variables
LS_HEAP_SIZE="${LS_HEAP_SIZE:=500m}"

setup_java() {
if [ -z "$JAVACMD" ] ; then
Expand Down Expand Up @@ -57,7 +62,7 @@ setup_java() {
}

setup_drip() {
if [ -z $DRIP_JAVACMD ] ; then
if [ -z "$DRIP_JAVACMD" ] ; then
JAVACMD="drip"
fi

Expand All @@ -74,7 +79,7 @@ setup_drip() {
# faster JRuby startup options https://github.com/jruby/jruby/wiki/Improving-startup-time
# since we are using drip to speed up, we may as well throw these in also
if [ "$USE_RUBY" = "1" ] ; then
export JRUBY_OPTS="-J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-noverify"
export JRUBY_OPTS="$JRUBY_OPTS -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-noverify"
else
JAVA_OPTS="$JAVA_OPTS -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -noverify"
fi
Expand All @@ -84,8 +89,7 @@ setup_drip() {
}

setup_vendored_jruby() {
#JRUBY_JAR=$(ls "${basedir}"/vendor/jruby/jruby-complete-*.jar)
JRUBY_BIN="${basedir}/vendor/jruby/bin/jruby"
JRUBY_BIN="${LOGSTASH_HOME}/vendor/jruby/bin/jruby"

if [ ! -f "${JRUBY_BIN}" ] ; then
echo "Unable to find JRuby."
Expand All @@ -102,9 +106,9 @@ setup_ruby() {
}

jruby_opts() {
echo "--1.9"
printf "%s" "--1.9"
for i in $JAVA_OPTS ; do
echo "-J$i"
printf "%s" " -J$i"
done
}

Expand All @@ -127,6 +131,24 @@ setup() {
setup_java
setup_vendored_jruby
fi
}

ruby_exec() {
if [ -z "$VENDORED_JRUBY" ] ; then

# $VENDORED_JRUBY is empty so use the local "ruby" command

if [ "$DEBUG" ] ; then
echo "DEBUG: exec ${RUBYCMD} $@"
fi
exec "${RUBYCMD}" "$@"
else

# $VENDORED_JRUBY is non-empty so use the vendored JRuby

export RUBYLIB="${basedir}/lib"
if [ "$DEBUG" ] ; then
echo "DEBUG: exec ${JRUBY_BIN} $(jruby_opts) $@"
fi
exec "${JRUBY_BIN}" $(jruby_opts) "$@"
fi
}
38 changes: 5 additions & 33 deletions bin/plugin
Original file line number Diff line number Diff line change
@@ -1,38 +1,10 @@
#!/bin/sh
# Run logstash from source
#
# This is most useful when done from a git checkout.
#
# Usage:
# bin/logstash <command> [arguments]
#
# See 'bin/logstash help' for a list of commands.
#
# Defaults you can override with environment variables
LS_HEAP_SIZE="${LS_HEAP_SIZE:=500m}"

unset CDPATH
basedir=$(cd `dirname $0`/..; pwd)
. "${basedir}/bin/logstash.lib.sh"

. "$(cd `dirname $0`/..; pwd)/bin/logstash.lib.sh"
setup

# Export these so that they can be picked up by file input (and others?).
export HOME SINCEDB_DIR
# bin/plugin is a short lived ruby script thus we can use aggressive "faster starting JRuby options"
# see https://github.com/jruby/jruby/wiki/Improving-startup-time
export JRUBY_OPTS="$JRUBY_OPTS -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-noverify -X-C -Xcompile.invokedynamic=false"

case $1 in
-*)
if [ -z "$VENDORED_JRUBY" ] ; then
exec "${RUBYCMD}" "${basedir}/lib/logstash/pluginmanager.rb" "$@"
else
exec "$JRUBY_BIN" $(jruby_opts) "${basedir}/lib/logstash/pluginmanager.rb" "$@"
fi
;;
*)
if [ -z "$VENDORED_JRUBY" ] ; then
exec "${RUBYCMD}" "${basedir}/lib/logstash/pluginmanager.rb" "$@"
else
exec "$JRUBY_BIN" $(jruby_opts) "${basedir}/lib/logstash/pluginmanager.rb" "$@"
fi
;;
esac
ruby_exec "${LOGSTASH_HOME}/lib/pluginmanager/main.rb" "$@"
Loading

0 comments on commit 48bcc5b

Please sign in to comment.