diff --git a/Gemfile b/Gemfile index cef2642798b..70d17c8350e 100644 --- a/Gemfile +++ b/Gemfile @@ -2,7 +2,9 @@ # If you modify this file manually all comments and formatting will be lost. source "https://rubygems.org" -gem "logstash-core", "3.0.0.dev", :path => "." +gem "logstash-core", "3.0.0.dev", :path => "./logstash-core" +gem "logstash-core-event", "3.0.0.dev", :path => "./logstash-core-event" +# gem "logstash-core-event-java", "3.0.0.dev", :path => "./logstash-core-event-java" gem "file-dependencies", "0.1.6" gem "ci_reporter_rspec", "1.0.0", :group => :development gem "simplecov", :group => :development diff --git a/Gemfile.jruby-1.9.lock b/Gemfile.jruby-1.9.lock index a2accef2d25..8f0779a5968 100644 --- a/Gemfile.jruby-1.9.lock +++ b/Gemfile.jruby-1.9.lock @@ -1,14 +1,14 @@ PATH - remote: . + remote: ./logstash-core specs: logstash-core (3.0.0.dev-java) cabin (~> 0.7.0) clamp (~> 0.6.5) - concurrent-ruby (~> 0.9.1) + concurrent-ruby (= 0.9.1) filesize (= 0.0.4) gems (~> 0.8.3) i18n (= 0.6.9) - jrjackson (~> 0.3.5) + jrjackson (~> 0.3.6) jruby-openssl (>= 0.9.11) minitar (~> 0.5.4) pry (~> 0.10.1) @@ -16,17 +16,23 @@ PATH thread_safe (~> 0.3.5) treetop (< 1.5.0) +PATH + remote: ./logstash-core-event + specs: + logstash-core-event (3.0.0.dev-java) + logstash-core (>= 2.0.0.beta2, < 3.0.0) + GEM remote: https://rubygems.org/ specs: addressable (2.3.8) arr-pm (0.0.10) cabin (> 0) - backports (3.6.6) + backports (3.6.7) benchmark-ips (2.3.0) builder (3.2.2) - cabin (0.7.1) - childprocess (0.5.6) + cabin (0.7.2) + childprocess (0.5.7) ffi (~> 1.0, >= 1.0.11) ci_reporter (2.0.0) builder (>= 2.1.2) @@ -67,7 +73,7 @@ GEM domain_name (~> 0.5) i18n (0.6.9) insist (1.0.0) - jrjackson (0.3.5) + jrjackson (0.3.6) jruby-openssl (0.9.12-java) json (1.8.3-java) kramdown (1.9.0) @@ -84,11 +90,11 @@ GEM mime-types (2.6.2) minitar (0.5.4) multipart-post (2.0.0) - netrc (0.10.3) + netrc (0.11.0) octokit (3.8.0) sawyer (~> 0.6.0, >= 0.5.3) polyglot (0.3.5) - pry (0.10.2-java) + pry (0.10.3-java) coderay (~> 1.1.0) method_source (~> 0.8.1) slop (~> 3.4) @@ -147,6 +153,7 @@ DEPENDENCIES fpm (~> 1.3.3) gems (~> 0.8.3) logstash-core (= 3.0.0.dev)! + logstash-core-event (= 3.0.0.dev)! logstash-devutils (~> 0.0.15) octokit (= 3.8.0) rspec (~> 3.1.0) diff --git a/java/settings.gradle b/java/settings.gradle deleted file mode 100644 index 95022f43e02..00000000000 --- a/java/settings.gradle +++ /dev/null @@ -1,2 +0,0 @@ -rootProject.name = 'logstash' -include 'logstash-event' diff --git a/lib/bootstrap/environment.rb b/lib/bootstrap/environment.rb index 9f3e59f5b08..0206959a400 100644 --- a/lib/bootstrap/environment.rb +++ b/lib/bootstrap/environment.rb @@ -35,6 +35,15 @@ def ruby_engine def logstash_gem_home ::File.join(BUNDLE_DIR, ruby_engine, gem_ruby_version) end + + def vendor_path(path) + return ::File.join(LOGSTASH_HOME, "vendor", path) + end + + def pattern_path(path) + return ::File.join(LOGSTASH_HOME, "patterns", path) + end + end end diff --git a/lib/jruby_event/Rakefile b/lib/jruby_event/Rakefile deleted file mode 100644 index 114167b8b9a..00000000000 --- a/lib/jruby_event/Rakefile +++ /dev/null @@ -1,45 +0,0 @@ -begin - require 'ant' -rescue - puts("error: unable to load Ant, make sure Ant is installed, in your PATH and $ANT_HOME is defined properly") - puts("\nerror details:\n#{$!}") - exit(1) -end - -BASEDIR = File.expand_path("../../../", __FILE__) - -TARGET = File.join(BASEDIR, "out/production/main/") -SRC = File.join(BASEDIR, "src/main/java") -JAR = File.join(BASEDIR, "lib/jruby_event/jruby_event.jar") - -task :setup do - ant.mkdir 'dir' => TARGET - ant.path 'id' => 'classpath' do - fileset 'dir' => TARGET - end -end - -desc "compile Java classes" -task :build => [:setup] do |t, args| - require 'jruby/jrubyc' - ant.javac( - :srcdir => SRC, - :destdir => TARGET, - :classpathref => "classpath", - :debug => true, - :includeantruntime => "no", - :verbose => false, - :listfiles => true, - :source => "1.7", - :target => "1.7", - ) {} -end - -task :setup_jar do - ant.delete 'file' => JAR -end - -desc "build the jar" -task :jar => [:setup_jar, :build] do - ant.jar :basedir => TARGET, :destfile => JAR, :includes => "**/*.class" -end diff --git a/lib/jruby_event/jackson-core-asl-1.9.13.jar b/lib/jruby_event/jackson-core-asl-1.9.13.jar deleted file mode 100644 index bb4fe1da118..00000000000 Binary files a/lib/jruby_event/jackson-core-asl-1.9.13.jar and /dev/null differ diff --git a/lib/jruby_event/jackson-mapper-asl-1.9.13.jar b/lib/jruby_event/jackson-mapper-asl-1.9.13.jar deleted file mode 100644 index 0f2073fc7ea..00000000000 Binary files a/lib/jruby_event/jackson-mapper-asl-1.9.13.jar and /dev/null differ diff --git a/lib/jruby_event/jruby_event.gemspec b/lib/jruby_event/jruby_event.gemspec deleted file mode 100644 index 08813ad4dac..00000000000 --- a/lib/jruby_event/jruby_event.gemspec +++ /dev/null @@ -1,10 +0,0 @@ -Gem::Specification.new do |s| - s.name = "jruby_event" - s.version = "0.0.1" - s.summary = 'A helper Gem for using the Docker API' - s.description = 'This gem is intended to aid in using Docker images and containers, specifically with regards to integration testing in RSpec.' - s.authors = ['Aaron Mildenstein', 'Tal Levy'] - s.email = 'aaron@mildensteins.com' - s.homepage = 'http://github.com/untergeek/longshoreman' - s.licenses = ['Apache License (2.0)'] - s.require_paths = ['lib'] diff --git a/lib/jruby_event/jruby_event.jar b/lib/jruby_event/jruby_event.jar deleted file mode 100644 index 75b4fbd3a5e..00000000000 Binary files a/lib/jruby_event/jruby_event.jar and /dev/null differ diff --git a/lib/jruby_event/jruby_event.rb b/lib/jruby_event/jruby_event.rb deleted file mode 100644 index 177eff26fe4..00000000000 --- a/lib/jruby_event/jruby_event.rb +++ /dev/null @@ -1,7 +0,0 @@ -# encoding: utf-8 - -require "java" -require "cabin" -require_relative "../../java/logstash-event/build/libs/logstash-event-all.jar" -require "jruby_event_ext" -require "jruby_timestamp_ext" diff --git a/lib/logstash-event.rb b/lib/logstash-event.rb deleted file mode 100644 index 0f44322944b..00000000000 --- a/lib/logstash-event.rb +++ /dev/null @@ -1,2 +0,0 @@ -# encoding: utf-8 -require "logstash/event" diff --git a/lib/logstash/patches/bundler.rb b/lib/logstash/patches/bundler.rb deleted file mode 100644 index 25d93a09148..00000000000 --- a/lib/logstash/patches/bundler.rb +++ /dev/null @@ -1,36 +0,0 @@ -# encoding: utf-8 -# Bundler monkey patches -module ::Bundler - # Patch bundler to write a .lock file specific to the version of ruby. - # This keeps MRI/JRuby/RBX from conflicting over the Gemfile.lock updates - module SharedHelpers - def default_lockfile - ruby = "#{LogStash::Environment.ruby_engine}-#{LogStash::Environment.ruby_abi_version}" - Pathname.new("#{default_gemfile}.#{ruby}.lock") - end - end - - # Patch to prevent Bundler to save a .bundle/config file in the root - # of the application - class Settings - def set_key(key, value, hash, file) - key = key_for(key) - - unless hash[key] == value - hash[key] = value - hash.delete(key) if value.nil? - end - - value - end - end - - # Add the Bundler.reset! method which has been added in master but is not in 1.7.9. - class << self - unless self.method_defined?("reset!") - def reset! - @definition = nil - end - end - end -end diff --git a/lib/logstash/timestamp.rb b/lib/logstash/timestamp.rb deleted file mode 100644 index 33e89c5d4dd..00000000000 --- a/lib/logstash/timestamp.rb +++ /dev/null @@ -1,97 +0,0 @@ -# encoding: utf-8 -require "logstash/environment" -require "logstash/json" -require "forwardable" -require "date" -require "time" - -# module LogStash -# class TimestampParserError < StandardError; end -# -# class Timestamp -# extend Forwardable -# include Comparable -# -# def_delegators :@time, :tv_usec, :usec, :year, :iso8601, :to_i, :tv_sec, :to_f, :to_edn, :<=>, :+ -# -# attr_reader :time -# -# ISO8601_STRFTIME = "%04d-%02d-%02dT%02d:%02d:%02d.%06d%+03d:00".freeze -# ISO8601_PRECISION = 3 -# -# def initialize(time = Time.new) -# @time = time.utc -# end -# -# def self.at(*args) -# Timestamp.new(::Time.at(*args)) -# end -# -# def self.parse(*args) -# Timestamp.new(::Time.parse(*args)) -# end -# -# def self.now -# Timestamp.new(::Time.now) -# end -# -# # coerce tries different strategies based on the time object class to convert into a Timestamp. -# # @param [String, Time, Timestamp] time the time object to try coerce -# # @return [Timestamp, nil] Timestamp will be returned if successful otherwise nil -# # @raise [TimestampParserError] on String with invalid format -# def self.coerce(time) -# case time -# when String -# LogStash::Timestamp.parse_iso8601(time) -# when LogStash::Timestamp -# time -# when Time -# LogStash::Timestamp.new(time) -# else -# nil -# end -# end -# -# if LogStash::Environment.jruby? -# JODA_ISO8601_PARSER = org.joda.time.format.ISODateTimeFormat.dateTimeParser -# UTC = org.joda.time.DateTimeZone.forID("UTC") -# -# def self.parse_iso8601(t) -# millis = JODA_ISO8601_PARSER.parseMillis(t) -# LogStash::Timestamp.at(millis / 1000, (millis % 1000) * 1000) -# rescue => e -# raise(TimestampParserError, "invalid timestamp string #{t.inspect}, error=#{e.inspect}") -# end -# -# else -# -# def self.parse_iso8601(t) -# # warning, ruby's Time.parse is *really* terrible and slow. -# LogStash::Timestamp.new(::Time.parse(t)) -# rescue => e -# raise(TimestampParserError, "invalid timestamp string #{t.inspect}, error=#{e.inspect}") -# end -# end -# -# def utc -# @time.utc # modifies the receiver -# self -# end -# alias_method :gmtime, :utc -# -# def to_json(*args) -# # ignore arguments to respect accepted to_json method signature -# "\"" + to_iso8601 + "\"" -# end -# alias_method :inspect, :to_json -# -# def to_iso8601 -# @iso8601 ||= @time.iso8601(ISO8601_PRECISION) -# end -# alias_method :to_s, :to_iso8601 -# -# def -(value) -# @time - (value.is_a?(Timestamp) ? value.time : value) -# end -# end -# end diff --git a/lib/logstash/util/accessors.rb b/lib/logstash/util/accessors.rb deleted file mode 100644 index a8f51a146b5..00000000000 --- a/lib/logstash/util/accessors.rb +++ /dev/null @@ -1,124 +0,0 @@ -# encoding: utf-8 -require "logstash/namespace" -require "logstash/util" -require "thread_safe" - -# module LogStash::Util -# -# # PathCache is a singleton which globally caches the relation between a field reference and its -# # decomposition into a [key, path array] tuple. For example the field reference [foo][bar][baz] -# # is decomposed into ["baz", ["foo", "bar"]]. -# module PathCache -# extend self -# -# # requiring libraries and defining constants is thread safe in JRuby so -# # PathCache::CACHE will be corretly initialized, once, when accessors.rb -# # will be first required -# CACHE = ThreadSafe::Cache.new -# -# def get(field_reference) -# # the "get_or_default(x, nil) || put(x, parse(x))" is ~2x faster than "get || put" because the get call is -# # proxied through the JRuby JavaProxy op_aref method. the correct idiom here would be to use -# # "compute_if_absent(x){parse(x)}" but because of the closure creation, it is ~1.5x slower than -# # "get_or_default || put". -# # this "get_or_default || put" is obviously non-atomic which is not really important here -# # since all threads will set the same value and this cache will stabilize very quickly after the first -# # few events. -# CACHE.get_or_default(field_reference, nil) || CACHE.put(field_reference, parse(field_reference)) -# end -# -# def parse(field_reference) -# path = field_reference.split(/[\[\]]/).select{|s| !s.empty?} -# [path.pop, path] -# end -# end -# -# # Accessors uses a lookup table to speedup access of a field reference of the form -# # "[hello][world]" to the underlying store hash into {"hello" => {"world" => "foo"}} -# class Accessors -# -# # @param store [Hash] the backing data store field refereces point to -# def initialize(store) -# @store = store -# -# # @lut is a lookup table between a field reference and a [target, key] tuple -# # where target is the containing Hash or Array for key in @store. -# # this allows us to directly access the containing object for key instead of -# # walking the field reference path into the inner @store objects -# @lut = {} -# end -# -# # @param field_reference [String] the field reference -# # @return [Object] the value in @store for this field reference -# def get(field_reference) -# target, key = lookup(field_reference) -# return nil unless target -# target.is_a?(Array) ? target[key.to_i] : target[key] -# end -# -# # @param field_reference [String] the field reference -# # @param value [Object] the value to set in @store for this field reference -# # @return [Object] the value set -# def set(field_reference, value) -# target, key = lookup_or_create(field_reference) -# target[target.is_a?(Array) ? key.to_i : key] = value -# end -# -# # @param field_reference [String] the field reference to remove -# # @return [Object] the removed value in @store for this field reference -# def del(field_reference) -# target, key = lookup(field_reference) -# return nil unless target -# target.is_a?(Array) ? target.delete_at(key.to_i) : target.delete(key) -# end -# -# # @param field_reference [String] the field reference to test for inclusion in the store -# # @return [Boolean] true if the store contains a value for this field reference -# def include?(field_reference) -# target, key = lookup(field_reference) -# return false unless target -# -# target.is_a?(Array) ? !target[key.to_i].nil? : target.include?(key) -# end -# -# private -# -# # retrieve the [target, key] tuple associated with this field reference -# # @param field_reference [String] the field referece -# # @return [[Object, String]] the [target, key] tuple associated with this field reference -# def lookup(field_reference) -# @lut[field_reference] ||= find_target(field_reference) -# end -# -# # retrieve the [target, key] tuple associated with this field reference and create inner -# # container objects if they do not exists -# # @param field_reference [String] the field referece -# # @return [[Object, String]] the [target, key] tuple associated with this field reference -# def lookup_or_create(field_reference) -# @lut[field_reference] ||= find_or_create_target(field_reference) -# end -# -# # find the target container object in store for this field reference -# # @param field_reference [String] the field referece -# # @return [Object] the target container object in store associated with this field reference -# def find_target(field_reference) -# key, path = PathCache.get(field_reference) -# target = path.inject(@store) do |r, k| -# return nil unless r -# r[r.is_a?(Array) ? k.to_i : k] -# end -# target ? [target, key] : nil -# end -# -# # find the target container object in store for this field reference and create inner -# # container objects if they do not exists -# # @param field_reference [String] the field referece -# # @return [Object] the target container object in store associated with this field reference -# def find_or_create_target(accessor) -# key, path = PathCache.get(accessor) -# target = path.inject(@store) {|r, k| r[r.is_a?(Array) ? k.to_i : k] ||= {}} -# [target, key] -# end -# end # class Accessors -# end # module LogStash::Util - diff --git a/java/.gitignore b/logstash-core-event-java/.gitignore similarity index 100% rename from java/.gitignore rename to logstash-core-event-java/.gitignore diff --git a/logstash-core-event-java/README.md b/logstash-core-event-java/README.md new file mode 100644 index 00000000000..7b12d19f135 --- /dev/null +++ b/logstash-core-event-java/README.md @@ -0,0 +1,63 @@ +# logstash-core-event-java + +## dev install + +1- build code with + +``` +$ cd logstash-core-event-java +$ gradle build +``` + +A bunch of warning are expected, it should end with: + +``` +BUILD SUCCESSFUL +``` + +2- update root logstash `Gemfile` to use this gem with: + +``` +# gem "logstash-core-event", "x.y.z", :path => "./logstash-core-event" +gem "logstash-core-event-java", "x.y.z", :path => "./logstash-core-event-java" +``` + +3- update `logstash-core/logstash-core.gemspec` with: + +``` +# gem.add_runtime_dependency "logstash-core-event", "x.y.z" +gem.add_runtime_dependency "logstash-core-event-java", "x.y.z" +``` + +4- and install: + +``` +$ bin/bundle +``` + +- install core plugins for tests + +``` +$ rake test:install-core +``` + +## specs + +``` +$ bin/rspec spec +$ bin/rspec logstash-core/spec +$ bin/rspec logstash-core-event/spec +$ bin/rspec logstash-core-event-java/spec +``` + +or + +``` +$ rake test:core +``` + +also + +``` +$ rake test:plugins +``` \ No newline at end of file diff --git a/java/build.gradle b/logstash-core-event-java/build.gradle similarity index 94% rename from java/build.gradle rename to logstash-core-event-java/build.gradle index e7185d90e33..6e4c299a79d 100644 --- a/java/build.gradle +++ b/logstash-core-event-java/build.gradle @@ -10,7 +10,8 @@ buildscript { } } -allprojects { +//allprojects { + repositories { mavenLocal() mavenCentral() @@ -22,9 +23,10 @@ allprojects { } } -} +//} + +//subprojects { project -> -subprojects { project -> apply plugin: 'java' apply plugin: 'idea' apply plugin: 'com.github.johnrengelman.shadow' @@ -87,15 +89,16 @@ subprojects { project -> compile 'joda-time:joda-time:2.8.2' compile 'com.google.guava:guava:18.0' compile 'org.slf4j:slf4j-api:1.7.12' - provided 'org.jruby:jruby-core:1.7.21' + provided 'org.jruby:jruby-core:1.7.22' testCompile 'org.testng:testng:6.9.6' testCompile 'org.mockito:mockito-core:1.10.19' } -} + +//} // See http://www.gradle.org/docs/current/userguide/gradle_wrapper.html task wrapper(type: Wrapper) { description = 'Install Gradle wrapper' - gradleVersion = '2.3' + gradleVersion = '2.7' } diff --git a/java/gradle.properties b/logstash-core-event-java/gradle.properties similarity index 100% rename from java/gradle.properties rename to logstash-core-event-java/gradle.properties diff --git a/java/gradle/wrapper/gradle-wrapper.jar b/logstash-core-event-java/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from java/gradle/wrapper/gradle-wrapper.jar rename to logstash-core-event-java/gradle/wrapper/gradle-wrapper.jar diff --git a/java/gradle/wrapper/gradle-wrapper.properties b/logstash-core-event-java/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from java/gradle/wrapper/gradle-wrapper.properties rename to logstash-core-event-java/gradle/wrapper/gradle-wrapper.properties diff --git a/java/gradlew b/logstash-core-event-java/gradlew similarity index 100% rename from java/gradlew rename to logstash-core-event-java/gradlew diff --git a/java/gradlew.bat b/logstash-core-event-java/gradlew.bat similarity index 100% rename from java/gradlew.bat rename to logstash-core-event-java/gradlew.bat diff --git a/logstash-core-event-java/lib/logstash-core-event-java.rb b/logstash-core-event-java/lib/logstash-core-event-java.rb new file mode 100644 index 00000000000..29b487aa192 --- /dev/null +++ b/logstash-core-event-java/lib/logstash-core-event-java.rb @@ -0,0 +1 @@ +require "logstash-core-event-java/logstash-core-event-java" \ No newline at end of file diff --git a/logstash-core-event-java/lib/logstash-core-event-java/logstash-core-event-java.rb b/logstash-core-event-java/lib/logstash-core-event-java/logstash-core-event-java.rb new file mode 100644 index 00000000000..7bf90a9bc0b --- /dev/null +++ b/logstash-core-event-java/lib/logstash-core-event-java/logstash-core-event-java.rb @@ -0,0 +1,31 @@ +# encoding: utf-8 + +require "java" + +module LogStash +end + +# TODO: (colin) integrate jar loading with gradle and verify dev vs prod environment setups + +# insert all jars in this directory into CLASSPATH +Dir.glob(File.join(File.expand_path("..", __FILE__), "*.jar")).each do |jar| + $CLASSPATH << jar unless $CLASSPATH.include?(jar) +end + +# TODO: (colin) correctly handle dev env build/ dir and local jar + +# local dev setup +classes_dir = File.expand_path("../../../build/classes/main", __FILE__) + +if File.directory?(classes_dir) + # if in local dev setup, add target to classpath + $CLASSPATH << classes_dir unless $CLASSPATH.include?(classes_dir) +else + # otherwise use included jar + raise("TODO build dir not found and no jar file") +end + +require "jruby_event_ext" +require "jruby_timestamp_ext" +require "logstash/event" +require "logstash/timestamp" \ No newline at end of file diff --git a/logstash-core-event-java/lib/logstash-core-event-java/version.rb b/logstash-core-event-java/lib/logstash-core-event-java/version.rb new file mode 100644 index 00000000000..6c297b7c2fd --- /dev/null +++ b/logstash-core-event-java/lib/logstash-core-event-java/version.rb @@ -0,0 +1,8 @@ +# encoding: utf-8 + +# The version of logstash core event java gem. +# +# Note to authors: this should not include dashes because 'gem' barfs if +# you include a dash in the version string. + +LOGSTASH_CORE_EVENT_JAVA_VERSION = "3.0.0.dev" diff --git a/logstash-core-event-java/lib/logstash-core-event.rb b/logstash-core-event-java/lib/logstash-core-event.rb new file mode 100644 index 00000000000..29b487aa192 --- /dev/null +++ b/logstash-core-event-java/lib/logstash-core-event.rb @@ -0,0 +1 @@ +require "logstash-core-event-java/logstash-core-event-java" \ No newline at end of file diff --git a/lib/logstash/event.rb b/logstash-core-event-java/lib/logstash/event.rb similarity index 69% rename from lib/logstash/event.rb rename to logstash-core-event-java/lib/logstash/event.rb index 92e407ed7b6..a2e6950885d 100644 --- a/lib/logstash/event.rb +++ b/logstash-core-event-java/lib/logstash/event.rb @@ -1,14 +1,7 @@ # encoding: utf-8 -require "jruby_event/jruby_event" -# require "time" -# require "date" -# require "cabin" require "logstash/namespace" -# require "logstash/util/accessors" -# require "logstash/timestamp" require "logstash/json" -require "logstash/string_interpolation" # transcient pipeline events for normal in-flow signaling as opposed to # flow altering exceptions. for now having base classes is adequate and @@ -24,3 +17,8 @@ module LogStash # LogStash::SHUTDOWN is used by plugins SHUTDOWN = LogStash::ShutdownEvent.new end + +# for backward compatibility, require "logstash/event" is used a lots of places so let's bootstrap the +# Java code loading from here. +# TODO: (colin) I think we should mass replace require "logstash/event" with require "logstash-core-event" +require "logstash-core-event" \ No newline at end of file diff --git a/logstash-core-event-java/lib/logstash/timestamp.rb b/logstash-core-event-java/lib/logstash/timestamp.rb new file mode 100644 index 00000000000..0a4661a2d19 --- /dev/null +++ b/logstash-core-event-java/lib/logstash/timestamp.rb @@ -0,0 +1,28 @@ +# encoding: utf-8 + +require "logstash/namespace" +require "logstash-core-event" + +module LogStash + class TimestampParserError < StandardError; end + + class Timestamp + include Comparable + + # TODO (colin) implement in Java + def <=>(other) + self.time <=> other.time + end + + # TODO (colin) implement in Java + def +(other) + self.time + other + end + + # TODO (colin) implement in Java + def -(value) + self.time - (value.is_a?(Timestamp) ? value.time : value) + end + + end +end diff --git a/logstash-core-event-java/logstash-core-event-java.gemspec b/logstash-core-event-java/logstash-core-event-java.gemspec new file mode 100644 index 00000000000..77667f66e40 --- /dev/null +++ b/logstash-core-event-java/logstash-core-event-java.gemspec @@ -0,0 +1,23 @@ +# -*- encoding: utf-8 -*- +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'logstash-core-event-java/version' + +Gem::Specification.new do |gem| + gem.authors = ["Jordan Sissel", "Pete Fritchman", "Elasticsearch"] + gem.email = ["jls@semicomplete.com", "petef@databits.net", "info@elasticsearch.com"] + gem.description = %q{The core event component of logstash, the scalable log and event management tool} + gem.summary = %q{logstash-core-event-java - The core event component of logstash} + gem.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" + gem.license = "Apache License (2.0)" + + gem.files = Dir.glob(["logstash-core-event-java.gemspec", "lib/**/*.rb", "spec/**/*.rb"]) + gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) + gem.name = "logstash-core-event-java" + gem.require_paths = ["lib"] + gem.version = LOGSTASH_CORE_EVENT_JAVA_VERSION + + if RUBY_PLATFORM == 'java' + gem.platform = RUBY_PLATFORM + end +end diff --git a/logstash-core-event-java/settings.gradle b/logstash-core-event-java/settings.gradle new file mode 100644 index 00000000000..3885bfa1686 --- /dev/null +++ b/logstash-core-event-java/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'logstash-core-event-java' + diff --git a/spec/jruby_event/event_spec.rb b/logstash-core-event-java/spec/event_spec.rb similarity index 81% rename from spec/jruby_event/event_spec.rb rename to logstash-core-event-java/spec/event_spec.rb index 0dcdfcd2a04..a4866ad21db 100644 --- a/spec/jruby_event/event_spec.rb +++ b/logstash-core-event-java/spec/event_spec.rb @@ -1,26 +1,27 @@ -$LOAD_PATH << File.expand_path("../../../lib", __FILE__) +# encoding: utf-8 -require "jruby_event/jruby_event" +require "spec_helper" require "logstash/util" require "logstash/event" +require "json" TIMESTAMP = "@timestamp" describe LogStash::Event do context "to_json" do - it "should serialize snmple values" do + it "should serialize simple values" do e = LogStash::Event.new({"foo" => "bar", "bar" => 1, "baz" => 1.0, TIMESTAMP => "2015-05-28T23:02:05.350Z"}) - expect(e.to_json).to eq("{\"foo\":\"bar\",\"bar\":1,\"baz\":1.0,\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}") + expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"foo\":\"bar\",\"bar\":1,\"baz\":1.0,\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}")) end it "should serialize deep hash values" do e = LogStash::Event.new({"foo" => {"bar" => 1, "baz" => 1.0, "biz" => "boz"}, TIMESTAMP => "2015-05-28T23:02:05.350Z"}) - expect(e.to_json).to eq("{\"foo\":{\"bar\":1,\"baz\":1.0,\"biz\":\"boz\"},\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}") + expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"foo\":{\"bar\":1,\"baz\":1.0,\"biz\":\"boz\"},\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}")) end it "should serialize deep array values" do e = LogStash::Event.new({"foo" => ["bar", 1, 1.0], TIMESTAMP => "2015-05-28T23:02:05.350Z"}) - expect(e.to_json).to eq("{\"foo\":[\"bar\",1,1.0],\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}") + expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"foo\":[\"bar\",1,1.0],\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}")) end it "should serialize deep hash from field reference assignments" do @@ -29,7 +30,7 @@ e["bar"] = 1 e["baz"] = 1.0 e["[fancy][pants][socks]"] = "shoes" - expect(e.to_json).to eq("{\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\",\"foo\":\"bar\",\"bar\":1,\"baz\":1.0,\"fancy\":{\"pants\":{\"socks\":\"shoes\"}}}") + expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\",\"foo\":\"bar\",\"bar\":1,\"baz\":1.0,\"fancy\":{\"pants\":{\"socks\":\"shoes\"}}}")) end end @@ -119,7 +120,7 @@ end context "append" do - it "show append" do + it "should append" do event = LogStash::Event.new("message" => "hello world") event.append(LogStash::Event.new("message" => "another thing")) expect(event["message"]).to eq(["hello world", "another thing"]) @@ -129,10 +130,9 @@ context "tags" do it "should tag" do event = LogStash::Event.new("message" => "hello world") - tag = "foo" - event["tags"] = [] - event["tags"] << tag unless event["tags"].include?(tag) + expect(event["tags"]).to be_nil + event["tags"] = ["foo"] + expect(event["tags"]).to eq(["foo"]) end end end -`` \ No newline at end of file diff --git a/spec/jruby_event/timestamp_spec.rb b/logstash-core-event-java/spec/timestamp_spec.rb similarity index 71% rename from spec/jruby_event/timestamp_spec.rb rename to logstash-core-event-java/spec/timestamp_spec.rb index 0e796ca0a0d..1c092696389 100644 --- a/spec/jruby_event/timestamp_spec.rb +++ b/logstash-core-event-java/spec/timestamp_spec.rb @@ -1,6 +1,7 @@ -$LOAD_PATH << File.expand_path("../../../lib", __FILE__) +# encoding: utf-8 -require "jruby_event/jruby_event" +require "spec_helper" +require "logstash/timestamp" describe LogStash::Timestamp do context "constructors" do @@ -19,7 +20,10 @@ expect(t.time.to_i).to eq(now.to_i) end + it "should raise exception on invalid format" do + expect{LogStash::Timestamp.new("foobar")}.to raise_error + end + end end -`` \ No newline at end of file diff --git a/java/logstash-event/src/main/java/JrubyEventExtService.java b/logstash-core-event-java/src/main/java/JrubyEventExtService.java similarity index 100% rename from java/logstash-event/src/main/java/JrubyEventExtService.java rename to logstash-core-event-java/src/main/java/JrubyEventExtService.java diff --git a/java/logstash-event/src/main/java/JrubyTimestampExtService.java b/logstash-core-event-java/src/main/java/JrubyTimestampExtService.java similarity index 100% rename from java/logstash-event/src/main/java/JrubyTimestampExtService.java rename to logstash-core-event-java/src/main/java/JrubyTimestampExtService.java diff --git a/java/logstash-event/src/main/java/com/logstash/Accessors.java b/logstash-core-event-java/src/main/java/com/logstash/Accessors.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/Accessors.java rename to logstash-core-event-java/src/main/java/com/logstash/Accessors.java diff --git a/logstash-core-event-java/src/main/java/com/logstash/Cloner.java b/logstash-core-event-java/src/main/java/com/logstash/Cloner.java new file mode 100644 index 00000000000..4823f10726a --- /dev/null +++ b/logstash-core-event-java/src/main/java/com/logstash/Cloner.java @@ -0,0 +1,56 @@ +package com.logstash; + +import java.util.*; + +public final class Cloner { + + private Cloner(){} + + public static T deep(final T input) { + if (input instanceof Map) { + return (T) deepMap((Map) input); + } else if (input instanceof List) { + return (T) deepList((List) input); + } else if (input instanceof Collection) { + throw new ClassCastException("unexpected Collection type " + input.getClass()); + } + + return input; + } + + private static List deepList(final List list) { + List clone; + if (list instanceof LinkedList) { + clone = new LinkedList(); + } else if (list instanceof ArrayList) { + clone = new ArrayList(); + } else { + throw new ClassCastException("unexpected List type " + list.getClass()); + } + + for (E item : list) { + clone.add(deep(item)); + } + + return clone; + } + + private static Map deepMap(final Map map) { + Map clone; + if (map instanceof LinkedHashMap) { + clone = new LinkedHashMap(); + } else if (map instanceof TreeMap) { + clone = new TreeMap(); + } else if (map instanceof HashMap) { + clone = new HashMap(); + } else { + throw new ClassCastException("unexpected Map type " + map.getClass()); + } + + for (Map.Entry entry : map.entrySet()) { + clone.put(entry.getKey(), deep(entry.getValue())); + } + + return clone; + } +} diff --git a/java/logstash-event/src/main/java/com/logstash/DateNode.java b/logstash-core-event-java/src/main/java/com/logstash/DateNode.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/DateNode.java rename to logstash-core-event-java/src/main/java/com/logstash/DateNode.java diff --git a/java/logstash-event/src/main/java/com/logstash/EpochNode.java b/logstash-core-event-java/src/main/java/com/logstash/EpochNode.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/EpochNode.java rename to logstash-core-event-java/src/main/java/com/logstash/EpochNode.java diff --git a/java/logstash-event/src/main/java/com/logstash/Event.java b/logstash-core-event-java/src/main/java/com/logstash/Event.java similarity index 97% rename from java/logstash-event/src/main/java/com/logstash/Event.java rename to logstash-core-event-java/src/main/java/com/logstash/Event.java index f561b764bd9..ccdb6d4eb2d 100644 --- a/java/logstash-event/src/main/java/com/logstash/Event.java +++ b/logstash-core-event-java/src/main/java/com/logstash/Event.java @@ -188,8 +188,10 @@ public String sprintf(String s) throws IOException { public Event clone() throws CloneNotSupportedException { - Event clone = (Event)super.clone(); - clone.setAccessors(new Accessors(clone.getData())); +// Event clone = (Event)super.clone(); +// clone.setAccessors(new Accessors(clone.getData())); + + Event clone = new Event(Cloner.deep(getData())); return clone; } diff --git a/java/logstash-event/src/main/java/com/logstash/FieldReference.java b/logstash-core-event-java/src/main/java/com/logstash/FieldReference.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/FieldReference.java rename to logstash-core-event-java/src/main/java/com/logstash/FieldReference.java diff --git a/java/logstash-event/src/main/java/com/logstash/KeyNode.java b/logstash-core-event-java/src/main/java/com/logstash/KeyNode.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/KeyNode.java rename to logstash-core-event-java/src/main/java/com/logstash/KeyNode.java diff --git a/java/logstash-event/src/main/java/com/logstash/PathCache.java b/logstash-core-event-java/src/main/java/com/logstash/PathCache.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/PathCache.java rename to logstash-core-event-java/src/main/java/com/logstash/PathCache.java diff --git a/java/logstash-event/src/main/java/com/logstash/RubyToJavaConverter.java b/logstash-core-event-java/src/main/java/com/logstash/RubyToJavaConverter.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/RubyToJavaConverter.java rename to logstash-core-event-java/src/main/java/com/logstash/RubyToJavaConverter.java diff --git a/java/logstash-event/src/main/java/com/logstash/StaticNode.java b/logstash-core-event-java/src/main/java/com/logstash/StaticNode.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/StaticNode.java rename to logstash-core-event-java/src/main/java/com/logstash/StaticNode.java diff --git a/java/logstash-event/src/main/java/com/logstash/StringInterpolation.java b/logstash-core-event-java/src/main/java/com/logstash/StringInterpolation.java similarity index 98% rename from java/logstash-event/src/main/java/com/logstash/StringInterpolation.java rename to logstash-core-event-java/src/main/java/com/logstash/StringInterpolation.java index 865c8f97ee3..77aea3e41d6 100644 --- a/java/logstash-event/src/main/java/com/logstash/StringInterpolation.java +++ b/logstash-core-event-java/src/main/java/com/logstash/StringInterpolation.java @@ -63,7 +63,7 @@ public TemplateNode compile(String template) { pos = matcher.end(); } - if(pos < template.length() - 1) { + if(pos <= template.length() - 1) { compiledTemplate.add(new StaticNode(template.substring(pos))); } } diff --git a/java/logstash-event/src/main/java/com/logstash/Template.java b/logstash-core-event-java/src/main/java/com/logstash/Template.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/Template.java rename to logstash-core-event-java/src/main/java/com/logstash/Template.java diff --git a/java/logstash-event/src/main/java/com/logstash/TemplateNode.java b/logstash-core-event-java/src/main/java/com/logstash/TemplateNode.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/TemplateNode.java rename to logstash-core-event-java/src/main/java/com/logstash/TemplateNode.java diff --git a/java/logstash-event/src/main/java/com/logstash/Timestamp.java b/logstash-core-event-java/src/main/java/com/logstash/Timestamp.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/Timestamp.java rename to logstash-core-event-java/src/main/java/com/logstash/Timestamp.java diff --git a/java/logstash-event/src/main/java/com/logstash/TimestampSerializer.java b/logstash-core-event-java/src/main/java/com/logstash/TimestampSerializer.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/TimestampSerializer.java rename to logstash-core-event-java/src/main/java/com/logstash/TimestampSerializer.java diff --git a/java/logstash-event/src/main/java/com/logstash/Util.java b/logstash-core-event-java/src/main/java/com/logstash/Util.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/Util.java rename to logstash-core-event-java/src/main/java/com/logstash/Util.java diff --git a/java/logstash-event/src/main/java/com/logstash/ext/JrubyEventExtLibrary.java b/logstash-core-event-java/src/main/java/com/logstash/ext/JrubyEventExtLibrary.java similarity index 100% rename from java/logstash-event/src/main/java/com/logstash/ext/JrubyEventExtLibrary.java rename to logstash-core-event-java/src/main/java/com/logstash/ext/JrubyEventExtLibrary.java diff --git a/java/logstash-event/src/main/java/com/logstash/ext/JrubyTimestampExtLibrary.java b/logstash-core-event-java/src/main/java/com/logstash/ext/JrubyTimestampExtLibrary.java similarity index 64% rename from java/logstash-event/src/main/java/com/logstash/ext/JrubyTimestampExtLibrary.java rename to logstash-core-event-java/src/main/java/com/logstash/ext/JrubyTimestampExtLibrary.java index c2b468da452..b84aadc9507 100644 --- a/java/logstash-event/src/main/java/com/logstash/ext/JrubyTimestampExtLibrary.java +++ b/logstash-core-event-java/src/main/java/com/logstash/ext/JrubyTimestampExtLibrary.java @@ -5,6 +5,7 @@ import org.jruby.*; import org.jruby.anno.JRubyClass; import org.jruby.anno.JRubyMethod; +import org.jruby.exceptions.RaiseException; import org.jruby.javasupport.JavaUtil; import org.jruby.runtime.Arity; import org.jruby.runtime.ObjectAllocator; @@ -79,6 +80,18 @@ public IRubyObject ruby_initialize(ThreadContext context, IRubyObject[] args) this.timestamp = new Timestamp(); } else if (time instanceof RubyTime) { this.timestamp = new Timestamp(((RubyTime)time).getDateTime()); + } else if (time instanceof RubyString) { + try { + this.timestamp = new Timestamp(((RubyString) time).toString()); + } catch (IllegalArgumentException e) { + throw new RaiseException( + getRuntime(), + getRuntime().getModule("LogStash").getClass("TimestampParserError"), + "invalid timestamp string format " + time, + true + ); + + } } else { throw context.runtime.newTypeError("wrong argument type " + time.getMetaClass() + " (expected Time)"); } @@ -121,6 +134,59 @@ public IRubyObject ruby_to_json(ThreadContext context, IRubyObject[] args) return RubyString.newString(context.runtime, "\"" + this.timestamp.toIso8601() + "\""); } + public static Timestamp newTimetsamp(IRubyObject time) + { + if (time.isNil()) { + return new Timestamp(); + } else if (time instanceof RubyTime) { + return new Timestamp(((RubyTime)time).getDateTime()); + } else if (time instanceof RubyString) { + return new Timestamp(((RubyString) time).toString()); + } else if (time instanceof RubyTimestamp) { + return new Timestamp(((RubyTimestamp) time).timestamp); + } else { + return null; + } + } + + + @JRubyMethod(name = "coerce", required = 1, meta = true) + public static IRubyObject ruby_coerce(ThreadContext context, IRubyObject recv, IRubyObject time) + { + try { + Timestamp ts = newTimetsamp(time); + return (ts == null) ? context.runtime.getNil() : RubyTimestamp.newRubyTimestamp(context.runtime, ts); + } catch (IllegalArgumentException e) { + throw new RaiseException( + context.runtime, + context.runtime.getModule("LogStash").getClass("TimestampParserError"), + "invalid timestamp format " + e.getMessage(), + true + ); + + } + } + + @JRubyMethod(name = "parse_iso8601", required = 1, meta = true) + public static IRubyObject ruby_parse_iso8601(ThreadContext context, IRubyObject recv, IRubyObject time) + { + if (time instanceof RubyString) { + try { + return RubyTimestamp.newRubyTimestamp(context.runtime, newTimetsamp(time)); + } catch (IllegalArgumentException e) { + throw new RaiseException( + context.runtime, + context.runtime.getModule("LogStash").getClass("TimestampParserError"), + "invalid timestamp format " + e.getMessage(), + true + ); + + } + } else { + throw context.runtime.newTypeError("wrong argument type " + time.getMetaClass() + " (expected String)"); + } + } + @JRubyMethod(name = "at", required = 1, optional = 1, meta = true) public static IRubyObject ruby_at(ThreadContext context, IRubyObject recv, IRubyObject[] args) { diff --git a/java/logstash-event/src/test/java/com/logstash/AccessorsTest.java b/logstash-core-event-java/src/test/java/com/logstash/AccessorsTest.java similarity index 100% rename from java/logstash-event/src/test/java/com/logstash/AccessorsTest.java rename to logstash-core-event-java/src/test/java/com/logstash/AccessorsTest.java diff --git a/java/logstash-event/src/test/java/com/logstash/EventTest.java b/logstash-core-event-java/src/test/java/com/logstash/EventTest.java similarity index 100% rename from java/logstash-event/src/test/java/com/logstash/EventTest.java rename to logstash-core-event-java/src/test/java/com/logstash/EventTest.java diff --git a/java/logstash-event/src/test/java/com/logstash/FieldReferenceTest.java b/logstash-core-event-java/src/test/java/com/logstash/FieldReferenceTest.java similarity index 100% rename from java/logstash-event/src/test/java/com/logstash/FieldReferenceTest.java rename to logstash-core-event-java/src/test/java/com/logstash/FieldReferenceTest.java diff --git a/java/logstash-event/src/test/java/com/logstash/StringInterpolationTest.java b/logstash-core-event-java/src/test/java/com/logstash/StringInterpolationTest.java similarity index 97% rename from java/logstash-event/src/test/java/com/logstash/StringInterpolationTest.java rename to logstash-core-event-java/src/test/java/com/logstash/StringInterpolationTest.java index 7352cdbd35f..52d4563db4b 100644 --- a/java/logstash-event/src/test/java/com/logstash/StringInterpolationTest.java +++ b/logstash-core-event-java/src/test/java/com/logstash/StringInterpolationTest.java @@ -2,6 +2,7 @@ import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.junit.Test; import java.io.IOException; @@ -97,7 +98,7 @@ public void TestEpoch() throws IOException { Event event = getTestEvent(); String path = "%{+%s}"; StringInterpolation si = StringInterpolation.getInstance(); - assertEquals("1443682800", si.evaluate(event, path)); + assertEquals("1443657600", si.evaluate(event, path)); } @Test @@ -132,7 +133,7 @@ public Event getTestEvent() { data.put("bar", "foo"); data.put("awesome", "logstash"); data.put("j", inner); - data.put("@timestamp", new DateTime(2015, 10, 1, 0, 0, 0)); + data.put("@timestamp", new DateTime(2015, 10, 1, 0, 0, 0, DateTimeZone.UTC)); Event event = new Event(data); diff --git a/java/logstash-event/src/test/java/com/logstash/TimestampTest.java b/logstash-core-event-java/src/test/java/com/logstash/TimestampTest.java similarity index 100% rename from java/logstash-event/src/test/java/com/logstash/TimestampTest.java rename to logstash-core-event-java/src/test/java/com/logstash/TimestampTest.java diff --git a/logstash-core-event/lib/logstash-core-event.rb b/logstash-core-event/lib/logstash-core-event.rb new file mode 100644 index 00000000000..b2979326dac --- /dev/null +++ b/logstash-core-event/lib/logstash-core-event.rb @@ -0,0 +1 @@ +require "logstash-core-event/logstash-core-event" \ No newline at end of file diff --git a/logstash-core-event/lib/logstash-core-event/logstash-core-event.rb b/logstash-core-event/lib/logstash-core-event/logstash-core-event.rb new file mode 100644 index 00000000000..b0f773e203c --- /dev/null +++ b/logstash-core-event/lib/logstash-core-event/logstash-core-event.rb @@ -0,0 +1,5 @@ +# encoding: utf-8 +module LogStash +end + +require "logstash/event" \ No newline at end of file diff --git a/logstash-core-event/lib/logstash-core-event/version.rb b/logstash-core-event/lib/logstash-core-event/version.rb new file mode 100644 index 00000000000..18e991d6b0c --- /dev/null +++ b/logstash-core-event/lib/logstash-core-event/version.rb @@ -0,0 +1,8 @@ +# encoding: utf-8 + +# The version of logstash core event gem. +# +# Note to authors: this should not include dashes because 'gem' barfs if +# you include a dash in the version string. + +LOGSTASH_CORE_EVENT_VERSION = "3.0.0.dev" diff --git a/logstash-core-event/lib/logstash/event.rb b/logstash-core-event/lib/logstash/event.rb new file mode 100644 index 00000000000..c00d5531305 --- /dev/null +++ b/logstash-core-event/lib/logstash/event.rb @@ -0,0 +1,275 @@ +# encoding: utf-8 +require "time" +require "date" +require "cabin" +require "logstash/namespace" +require "logstash/util/accessors" +require "logstash/timestamp" +require "logstash/json" +require "logstash/string_interpolation" + +# transcient pipeline events for normal in-flow signaling as opposed to +# flow altering exceptions. for now having base classes is adequate and +# in the future it might be necessary to refactor using like a BaseEvent +# class to have a common interface for all pileline events to support +# eventual queueing persistence for example, TBD. +class LogStash::ShutdownEvent; end +class LogStash::FlushEvent; end + +module LogStash + FLUSH = LogStash::FlushEvent.new + + # LogStash::SHUTDOWN is used by plugins + SHUTDOWN = LogStash::ShutdownEvent.new +end + +# the logstash event object. +# +# An event is simply a tuple of (timestamp, data). +# The 'timestamp' is an ISO8601 timestamp. Data is anything - any message, +# context, references, etc that are relevant to this event. +# +# Internally, this is represented as a hash with only two guaranteed fields. +# +# * "@timestamp" - an ISO8601 timestamp representing the time the event +# occurred at. +# * "@version" - the version of the schema. Currently "1" +# +# They are prefixed with an "@" symbol to avoid clashing with your +# own custom fields. +# +# When serialized, this is represented in JSON. For example: +# +# { +# "@timestamp": "2013-02-09T20:39:26.234Z", +# "@version": "1", +# message: "hello world" +# } +class LogStash::Event + class DeprecatedMethod < StandardError; end + + CHAR_PLUS = "+" + TIMESTAMP = "@timestamp" + VERSION = "@version" + VERSION_ONE = "1" + TIMESTAMP_FAILURE_TAG = "_timestampparsefailure" + TIMESTAMP_FAILURE_FIELD = "_@timestamp" + + METADATA = "@metadata".freeze + METADATA_BRACKETS = "[#{METADATA}]".freeze + + # Floats outside of these upper and lower bounds are forcibly converted + # to scientific notation by Float#to_s + MIN_FLOAT_BEFORE_SCI_NOT = 0.0001 + MAX_FLOAT_BEFORE_SCI_NOT = 1000000000000000.0 + + LOGGER = Cabin::Channel.get(LogStash) + + public + def initialize(data = {}) + @cancelled = false + @data = data + @accessors = LogStash::Util::Accessors.new(data) + @data[VERSION] ||= VERSION_ONE + ts = @data[TIMESTAMP] + @data[TIMESTAMP] = ts ? init_timestamp(ts) : LogStash::Timestamp.now + + @metadata = @data.delete(METADATA) || {} + @metadata_accessors = LogStash::Util::Accessors.new(@metadata) + end # def initialize + + public + def cancel + @cancelled = true + end # def cancel + + public + def uncancel + @cancelled = false + end # def uncancel + + public + def cancelled? + return @cancelled + end # def cancelled? + + # Create a deep-ish copy of this event. + public + def clone + copy = {} + @data.each do |k,v| + # TODO(sissel): Recurse if this is a hash/array? + copy[k] = begin v.clone rescue v end + end + return self.class.new(copy) + end # def clone + + public + def to_s + self.sprintf("#{timestamp.to_iso8601} %{host} %{message}") + end # def to_s + + public + def timestamp; return @data[TIMESTAMP]; end # def timestamp + def timestamp=(val); return @data[TIMESTAMP] = val; end # def timestamp= + + def unix_timestamp + raise DeprecatedMethod + end # def unix_timestamp + + def ruby_timestamp + raise DeprecatedMethod + end # def unix_timestamp + + public + def [](fieldref) + if fieldref.start_with?(METADATA_BRACKETS) + @metadata_accessors.get(fieldref[METADATA_BRACKETS.length .. -1]) + elsif fieldref == METADATA + @metadata + else + @accessors.get(fieldref) + end + end # def [] + + public + def []=(fieldref, value) + if fieldref == TIMESTAMP && !value.is_a?(LogStash::Timestamp) + raise TypeError, "The field '@timestamp' must be a (LogStash::Timestamp, not a #{value.class} (#{value})" + end + if fieldref.start_with?(METADATA_BRACKETS) + @metadata_accessors.set(fieldref[METADATA_BRACKETS.length .. -1], value) + elsif fieldref == METADATA + @metadata = value + @metadata_accessors = LogStash::Util::Accessors.new(@metadata) + else + @accessors.set(fieldref, value) + end + end # def []= + + public + def fields + raise DeprecatedMethod + end + + public + def to_json(*args) + # ignore arguments to respect accepted to_json method signature + LogStash::Json.dump(@data) + end # def to_json + + public + def to_hash + @data + end # def to_hash + + public + def overwrite(event) + # pickup new event @data and also pickup @accessors + # otherwise it will be pointing on previous data + @data = event.instance_variable_get(:@data) + @accessors = event.instance_variable_get(:@accessors) + + #convert timestamp if it is a String + if @data[TIMESTAMP].is_a?(String) + @data[TIMESTAMP] = LogStash::Timestamp.parse_iso8601(@data[TIMESTAMP]) + end + end + + public + def include?(fieldref) + if fieldref.start_with?(METADATA_BRACKETS) + @metadata_accessors.include?(fieldref[METADATA_BRACKETS.length .. -1]) + elsif fieldref == METADATA + true + else + @accessors.include?(fieldref) + end + end # def include? + + # Append an event to this one. + public + def append(event) + # non-destructively merge that event with ourselves. + + # no need to reset @accessors here because merging will not disrupt any existing field paths + # and if new ones are created they will be picked up. + LogStash::Util.hash_merge(@data, event.to_hash) + end # append + + # Remove a field or field reference. Returns the value of that field when + # deleted + public + def remove(fieldref) + @accessors.del(fieldref) + end # def remove + + # sprintf. This could use a better method name. + # The idea is to take an event and convert it to a string based on + # any format values, delimited by %{foo} where 'foo' is a field or + # metadata member. + # + # For example, if the event has type == "foo" and host == "bar" + # then this string: + # "type is %{type} and source is %{host}" + # will return + # "type is foo and source is bar" + # + # If a %{name} value is an array, then we will join by ',' + # If a %{name} value does not exist, then no substitution occurs. + public + def sprintf(format) + LogStash::StringInterpolation.evaluate(self, format) + end + + def tag(value) + # Generalize this method for more usability + self["tags"] ||= [] + self["tags"] << value unless self["tags"].include?(value) + end + + private + + def init_timestamp(o) + begin + timestamp = LogStash::Timestamp.coerce(o) + return timestamp if timestamp + + LOGGER.warn("Unrecognized #{TIMESTAMP} value, setting current time to #{TIMESTAMP}, original in #{TIMESTAMP_FAILURE_FIELD}field", :value => o.inspect) + rescue LogStash::TimestampParserError => e + LOGGER.warn("Error parsing #{TIMESTAMP} string, setting current time to #{TIMESTAMP}, original in #{TIMESTAMP_FAILURE_FIELD} field", :value => o.inspect, :exception => e.message) + end + + @data["tags"] ||= [] + @data["tags"] << TIMESTAMP_FAILURE_TAG unless @data["tags"].include?(TIMESTAMP_FAILURE_TAG) + @data[TIMESTAMP_FAILURE_FIELD] = o + + LogStash::Timestamp.now + end + + public + def to_hash_with_metadata + @metadata.empty? ? to_hash : to_hash.merge(METADATA => @metadata) + end + + public + def to_json_with_metadata(*args) + # ignore arguments to respect accepted to_json method signature + LogStash::Json.dump(to_hash_with_metadata) + end # def to_json + + def self.validate_value(value) + case value + when String + raise("expected UTF-8 encoding for value=#{value}, encoding=#{value.encoding.inspect}") unless value.encoding == Encoding::UTF_8 + raise("invalid UTF-8 encoding for value=#{value}, encoding=#{value.encoding.inspect}") unless value.valid_encoding? + value + when Array + value.each{|v| validate_value(v)} # don't map, return original object + value + else + value + end + end + +end # class LogStash::Event diff --git a/lib/logstash/string_interpolation.rb b/logstash-core-event/lib/logstash/string_interpolation.rb similarity index 100% rename from lib/logstash/string_interpolation.rb rename to logstash-core-event/lib/logstash/string_interpolation.rb diff --git a/logstash-core-event/lib/logstash/timestamp.rb b/logstash-core-event/lib/logstash/timestamp.rb new file mode 100644 index 00000000000..fb75c5f2538 --- /dev/null +++ b/logstash-core-event/lib/logstash/timestamp.rb @@ -0,0 +1,97 @@ +# encoding: utf-8 +require "logstash/environment" +require "logstash/json" +require "forwardable" +require "date" +require "time" + +module LogStash + class TimestampParserError < StandardError; end + + class Timestamp + extend Forwardable + include Comparable + + def_delegators :@time, :tv_usec, :usec, :year, :iso8601, :to_i, :tv_sec, :to_f, :to_edn, :<=>, :+ + + attr_reader :time + + ISO8601_STRFTIME = "%04d-%02d-%02dT%02d:%02d:%02d.%06d%+03d:00".freeze + ISO8601_PRECISION = 3 + + def initialize(time = Time.new) + @time = time.utc + end + + def self.at(*args) + Timestamp.new(::Time.at(*args)) + end + + def self.parse(*args) + Timestamp.new(::Time.parse(*args)) + end + + def self.now + Timestamp.new(::Time.now) + end + + # coerce tries different strategies based on the time object class to convert into a Timestamp. + # @param [String, Time, Timestamp] time the time object to try coerce + # @return [Timestamp, nil] Timestamp will be returned if successful otherwise nil + # @raise [TimestampParserError] on String with invalid format + def self.coerce(time) + case time + when String + LogStash::Timestamp.parse_iso8601(time) + when LogStash::Timestamp + time + when Time + LogStash::Timestamp.new(time) + else + nil + end + end + + if LogStash::Environment.jruby? + JODA_ISO8601_PARSER = org.joda.time.format.ISODateTimeFormat.dateTimeParser + UTC = org.joda.time.DateTimeZone.forID("UTC") + + def self.parse_iso8601(t) + millis = JODA_ISO8601_PARSER.parseMillis(t) + LogStash::Timestamp.at(millis / 1000, (millis % 1000) * 1000) + rescue => e + raise(TimestampParserError, "invalid timestamp string #{t.inspect}, error=#{e.inspect}") + end + + else + + def self.parse_iso8601(t) + # warning, ruby's Time.parse is *really* terrible and slow. + LogStash::Timestamp.new(::Time.parse(t)) + rescue => e + raise(TimestampParserError, "invalid timestamp string #{t.inspect}, error=#{e.inspect}") + end + end + + def utc + @time.utc # modifies the receiver + self + end + alias_method :gmtime, :utc + + def to_json(*args) + # ignore arguments to respect accepted to_json method signature + "\"" + to_iso8601 + "\"" + end + alias_method :inspect, :to_json + + def to_iso8601 + @iso8601 ||= @time.iso8601(ISO8601_PRECISION) + end + alias_method :to_s, :to_iso8601 + + def -(value) + @time - (value.is_a?(Timestamp) ? value.time : value) + end + end +end diff --git a/logstash-core-event/lib/logstash/util/accessors.rb b/logstash-core-event/lib/logstash/util/accessors.rb new file mode 100644 index 00000000000..01c16910855 --- /dev/null +++ b/logstash-core-event/lib/logstash/util/accessors.rb @@ -0,0 +1,123 @@ +# encoding: utf-8 +require "logstash/namespace" +require "logstash/util" +require "thread_safe" + +module LogStash::Util + + # PathCache is a singleton which globally caches the relation between a field reference and its + # decomposition into a [key, path array] tuple. For example the field reference [foo][bar][baz] + # is decomposed into ["baz", ["foo", "bar"]]. + module PathCache + extend self + + # requiring libraries and defining constants is thread safe in JRuby so + # PathCache::CACHE will be corretly initialized, once, when accessors.rb + # will be first required + CACHE = ThreadSafe::Cache.new + + def get(field_reference) + # the "get_or_default(x, nil) || put(x, parse(x))" is ~2x faster than "get || put" because the get call is + # proxied through the JRuby JavaProxy op_aref method. the correct idiom here would be to use + # "compute_if_absent(x){parse(x)}" but because of the closure creation, it is ~1.5x slower than + # "get_or_default || put". + # this "get_or_default || put" is obviously non-atomic which is not really important here + # since all threads will set the same value and this cache will stabilize very quickly after the first + # few events. + CACHE.get_or_default(field_reference, nil) || CACHE.put(field_reference, parse(field_reference)) + end + + def parse(field_reference) + path = field_reference.split(/[\[\]]/).select{|s| !s.empty?} + [path.pop, path] + end + end + + # Accessors uses a lookup table to speedup access of a field reference of the form + # "[hello][world]" to the underlying store hash into {"hello" => {"world" => "foo"}} + class Accessors + + # @param store [Hash] the backing data store field refereces point to + def initialize(store) + @store = store + + # @lut is a lookup table between a field reference and a [target, key] tuple + # where target is the containing Hash or Array for key in @store. + # this allows us to directly access the containing object for key instead of + # walking the field reference path into the inner @store objects + @lut = {} + end + + # @param field_reference [String] the field reference + # @return [Object] the value in @store for this field reference + def get(field_reference) + target, key = lookup(field_reference) + return nil unless target + target.is_a?(Array) ? target[key.to_i] : target[key] + end + + # @param field_reference [String] the field reference + # @param value [Object] the value to set in @store for this field reference + # @return [Object] the value set + def set(field_reference, value) + target, key = lookup_or_create(field_reference) + target[target.is_a?(Array) ? key.to_i : key] = value + end + + # @param field_reference [String] the field reference to remove + # @return [Object] the removed value in @store for this field reference + def del(field_reference) + target, key = lookup(field_reference) + return nil unless target + target.is_a?(Array) ? target.delete_at(key.to_i) : target.delete(key) + end + + # @param field_reference [String] the field reference to test for inclusion in the store + # @return [Boolean] true if the store contains a value for this field reference + def include?(field_reference) + target, key = lookup(field_reference) + return false unless target + + target.is_a?(Array) ? !target[key.to_i].nil? : target.include?(key) + end + + private + + # retrieve the [target, key] tuple associated with this field reference + # @param field_reference [String] the field referece + # @return [[Object, String]] the [target, key] tuple associated with this field reference + def lookup(field_reference) + @lut[field_reference] ||= find_target(field_reference) + end + + # retrieve the [target, key] tuple associated with this field reference and create inner + # container objects if they do not exists + # @param field_reference [String] the field referece + # @return [[Object, String]] the [target, key] tuple associated with this field reference + def lookup_or_create(field_reference) + @lut[field_reference] ||= find_or_create_target(field_reference) + end + + # find the target container object in store for this field reference + # @param field_reference [String] the field referece + # @return [Object] the target container object in store associated with this field reference + def find_target(field_reference) + key, path = PathCache.get(field_reference) + target = path.inject(@store) do |r, k| + return nil unless r + r[r.is_a?(Array) ? k.to_i : k] + end + target ? [target, key] : nil + end + + # find the target container object in store for this field reference and create inner + # container objects if they do not exists + # @param field_reference [String] the field referece + # @return [Object] the target container object in store associated with this field reference + def find_or_create_target(accessor) + key, path = PathCache.get(accessor) + target = path.inject(@store) {|r, k| r[r.is_a?(Array) ? k.to_i : k] ||= {}} + [target, key] + end + end # class Accessors +end # module LogStash::Util diff --git a/logstash-core-event/logstash-core-event.gemspec b/logstash-core-event/logstash-core-event.gemspec new file mode 100644 index 00000000000..5fcddccfdf0 --- /dev/null +++ b/logstash-core-event/logstash-core-event.gemspec @@ -0,0 +1,23 @@ +# -*- encoding: utf-8 -*- +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'logstash-core-event/version' + +Gem::Specification.new do |gem| + gem.authors = ["Jordan Sissel", "Pete Fritchman", "Elasticsearch"] + gem.email = ["jls@semicomplete.com", "petef@databits.net", "info@elasticsearch.com"] + gem.description = %q{The core event component of logstash, the scalable log and event management tool} + gem.summary = %q{logstash-core-event - The core event component of logstash} + gem.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" + gem.license = "Apache License (2.0)" + + gem.files = Dir.glob(["logstash-core-event.gemspec", "lib/**/*.rb", "spec/**/*.rb"]) + gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) + gem.name = "logstash-core-event" + gem.require_paths = ["lib"] + gem.version = LOGSTASH_CORE_EVENT_VERSION + + if RUBY_PLATFORM == 'java' + gem.platform = RUBY_PLATFORM + end +end diff --git a/spec/core/event_spec.rb b/logstash-core-event/spec/logstash/event_spec.rb similarity index 92% rename from spec/core/event_spec.rb rename to logstash-core-event/spec/logstash/event_spec.rb index e8b184de8a2..7a8c217658e 100644 --- a/spec/core/event_spec.rb +++ b/logstash-core-event/spec/logstash/event_spec.rb @@ -1,5 +1,6 @@ # encoding: utf-8 require "spec_helper" +require "json" describe LogStash::Event do @@ -184,7 +185,6 @@ "type" => "new", "message" => "foo bar", ) - subject.overwrite(new_event) expect(subject["message"]).to eq("foo bar") @@ -198,7 +198,7 @@ context "#append" do it "should append strings to an array" do - what = subject.append(LogStash::Event.new("message" => "another thing")) + subject.append(LogStash::Event.new("message" => "another thing")) expect(subject["message"]).to eq([ "hello world", "another thing" ]) end @@ -241,7 +241,6 @@ expect(subject[ "field1" ]).to eq([ "original1", "append1" ]) end end - context "when event field is an array" do before { subject[ "field1" ] = [ "original1", "original2" ] } @@ -319,46 +318,45 @@ it "should coerce timestamp" do t = Time.iso8601("2014-06-12T00:12:17.114Z") - # expect(LogStash::Timestamp).to receive(:coerce).exactly(3).times.and_call_original expect(LogStash::Event.new("@timestamp" => t).timestamp.to_i).to eq(t.to_i) expect(LogStash::Event.new("@timestamp" => LogStash::Timestamp.new(t)).timestamp.to_i).to eq(t.to_i) expect(LogStash::Event.new("@timestamp" => "2014-06-12T00:12:17.114Z").timestamp.to_i).to eq(t.to_i) end it "should assign current time when no timestamp" do - # ts = LogStash::Timestamp.now - # expect(LogStash::Timestamp).to receive(:now).and_return(ts) - expect(LogStash::Event.new({}).timestamp.to_i).to be_within(1).of Time.now.to_i + expect(LogStash::Event.new({}).timestamp.to_i).to be_within(1).of (Time.now.to_i) end - it "should tag and warn for invalid value" do - ts = LogStash::Timestamp.now - # TODO(talevy): make pass. bridge between error in Java to Ruby - # expect(LogStash::Timestamp).to receive(:now).twice.and_return(ts) - # expect(LogStash::Event::LOGGER).to receive(:warn).twice - + it "should tag for invalid value" do event = LogStash::Event.new("@timestamp" => :foo) - expect(event.timestamp.to_i).to eq(ts.to_i) + expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i expect(event["tags"]).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG]) expect(event[LogStash::Event::TIMESTAMP_FAILURE_FIELD]).to eq(:foo) event = LogStash::Event.new("@timestamp" => 666) - expect(event.timestamp.to_i).to eq(ts.to_i) + expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i expect(event["tags"]).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG]) expect(event[LogStash::Event::TIMESTAMP_FAILURE_FIELD]).to eq(666) end - it "should tag and warn for invalid string format" do - ts = LogStash::Timestamp.now - # TODO(talevy): make pass. bridge between error in Java to Ruby - # expect(LogStash::Timestamp).to receive(:now).and_return(ts) - # expect(LogStash::Event::LOGGER).to receive(:warn) + it "should warn for invalid value" do + expect(LogStash::Event::LOGGER).to receive(:warn).twice + + LogStash::Event.new("@timestamp" => :foo) + LogStash::Event.new("@timestamp" => 666) + end + it "should tag for invalid string format" do event = LogStash::Event.new("@timestamp" => "foo") - expect(event.timestamp.to_i).to eq(ts.to_i) + expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i expect(event["tags"]).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG]) expect(event[LogStash::Event::TIMESTAMP_FAILURE_FIELD]).to eq("foo") end + + it "should warn for invalid string format" do + expect(LogStash::Event::LOGGER).to receive(:warn) + LogStash::Event.new("@timestamp" => "foo") + end end context "to_json" do @@ -369,7 +367,7 @@ ) json = new_event.to_json - expect(json).to eq( "{\"@timestamp\":\"2014-09-23T19:26:15.832Z\",\"@version\":\"1\",\"message\":\"foo bar\"}") + expect(JSON.parse(json)).to eq( JSON.parse("{\"@timestamp\":\"2014-09-23T19:26:15.832Z\",\"message\":\"foo bar\",\"@version\":\"1\"}")) end it "should support to_json and ignore arguments" do @@ -379,7 +377,7 @@ ) json = new_event.to_json(:foo => 1, :bar => "baz") - expect(json).to eq( "{\"@timestamp\":\"2014-09-23T19:26:15.832Z\",\"@version\":\"1\",\"message\":\"foo bar\"}") + expect(JSON.parse(json)).to eq( JSON.parse("{\"@timestamp\":\"2014-09-23T19:26:15.832Z\",\"message\":\"foo bar\",\"@version\":\"1\"}")) end end diff --git a/spec/core/timestamp_spec.rb b/logstash-core-event/spec/logstash/timestamp_spec.rb similarity index 100% rename from spec/core/timestamp_spec.rb rename to logstash-core-event/spec/logstash/timestamp_spec.rb diff --git a/spec/util/accessors_spec.rb b/logstash-core-event/spec/logstash/util/accessors_spec.rb similarity index 93% rename from spec/util/accessors_spec.rb rename to logstash-core-event/spec/logstash/util/accessors_spec.rb index ffbd67ee655..e3c1a73e60e 100644 --- a/spec/util/accessors_spec.rb +++ b/logstash-core-event/spec/logstash/util/accessors_spec.rb @@ -1,8 +1,17 @@ # encoding: utf-8 require "spec_helper" -require "logstash/util/accessors" -describe LogStash::Util::Accessors, :if => true dogit ad +# this is to skip specs when running agains an alternate logstash-core-event implementation +# that does not define the Accessors class. For example, in logstash-core-event-java +# the Accessors class does not exists in the Ruby namespace. +class_exists = begin + require "logstash/util/accessors" + true +rescue LoadError + false +end + +describe "LogStash::Util::Accessors", :if => class_exists do context "using simple field" do diff --git a/logstash-core/lib/logstash-core.rb b/logstash-core/lib/logstash-core.rb new file mode 100644 index 00000000000..c2e4557afa8 --- /dev/null +++ b/logstash-core/lib/logstash-core.rb @@ -0,0 +1 @@ +require "logstash-core/logstash-core" diff --git a/lib/logstash-core.rb b/logstash-core/lib/logstash-core/logstash-core.rb similarity index 100% rename from lib/logstash-core.rb rename to logstash-core/lib/logstash-core/logstash-core.rb diff --git a/lib/logstash/version.rb b/logstash-core/lib/logstash-core/version.rb similarity index 64% rename from lib/logstash/version.rb rename to logstash-core/lib/logstash-core/version.rb index 17a5cd8c15b..fdc9d13f1a4 100644 --- a/lib/logstash/version.rb +++ b/logstash-core/lib/logstash-core/version.rb @@ -1,6 +1,8 @@ # encoding: utf-8 -# The version of logstash. -LOGSTASH_VERSION = "3.0.0.dev" +# The version of logstash core gem. +# # Note to authors: this should not include dashes because 'gem' barfs if # you include a dash in the version string. + +LOGSTASH_CORE_VERSION = "3.0.0.dev" diff --git a/lib/logstash/agent.rb b/logstash-core/lib/logstash/agent.rb similarity index 100% rename from lib/logstash/agent.rb rename to logstash-core/lib/logstash/agent.rb diff --git a/lib/logstash/certs/cacert.pem b/logstash-core/lib/logstash/certs/cacert.pem similarity index 100% rename from lib/logstash/certs/cacert.pem rename to logstash-core/lib/logstash/certs/cacert.pem diff --git a/lib/logstash/codecs/base.rb b/logstash-core/lib/logstash/codecs/base.rb similarity index 100% rename from lib/logstash/codecs/base.rb rename to logstash-core/lib/logstash/codecs/base.rb diff --git a/lib/logstash/config/config_ast.rb b/logstash-core/lib/logstash/config/config_ast.rb similarity index 100% rename from lib/logstash/config/config_ast.rb rename to logstash-core/lib/logstash/config/config_ast.rb diff --git a/lib/logstash/config/cpu_core_strategy.rb b/logstash-core/lib/logstash/config/cpu_core_strategy.rb similarity index 100% rename from lib/logstash/config/cpu_core_strategy.rb rename to logstash-core/lib/logstash/config/cpu_core_strategy.rb diff --git a/lib/logstash/config/defaults.rb b/logstash-core/lib/logstash/config/defaults.rb similarity index 100% rename from lib/logstash/config/defaults.rb rename to logstash-core/lib/logstash/config/defaults.rb diff --git a/lib/logstash/config/file.rb b/logstash-core/lib/logstash/config/file.rb similarity index 100% rename from lib/logstash/config/file.rb rename to logstash-core/lib/logstash/config/file.rb diff --git a/lib/logstash/config/grammar.rb b/logstash-core/lib/logstash/config/grammar.rb similarity index 100% rename from lib/logstash/config/grammar.rb rename to logstash-core/lib/logstash/config/grammar.rb diff --git a/lib/logstash/config/grammar.treetop b/logstash-core/lib/logstash/config/grammar.treetop similarity index 100% rename from lib/logstash/config/grammar.treetop rename to logstash-core/lib/logstash/config/grammar.treetop diff --git a/lib/logstash/config/mixin.rb b/logstash-core/lib/logstash/config/mixin.rb similarity index 100% rename from lib/logstash/config/mixin.rb rename to logstash-core/lib/logstash/config/mixin.rb diff --git a/lib/logstash/config/registry.rb b/logstash-core/lib/logstash/config/registry.rb similarity index 100% rename from lib/logstash/config/registry.rb rename to logstash-core/lib/logstash/config/registry.rb diff --git a/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb similarity index 81% rename from lib/logstash/environment.rb rename to logstash-core/lib/logstash/environment.rb index 8f710eed088..79e7f24d86c 100644 --- a/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -1,18 +1,10 @@ # encoding: utf-8 require "logstash/errors" -require "logstash/version" module LogStash module Environment extend self - # rehydrate the bootstrap environment if the startup was not done by executing bootstrap.rb - # and we are in the context of the logstash package - if !LogStash::Environment.const_defined?("LOGSTASH_HOME") && !ENV["LOGSTASH_HOME"].to_s.empty? - $LOAD_PATH << ::File.join(ENV["LOGSTASH_HOME"], "lib") - require "bootstrap/environment" - end - LOGSTASH_CORE = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", "..")) LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze @@ -81,14 +73,6 @@ def windows? ::Gem.win_platform? end - def vendor_path(path) - return ::File.join(LOGSTASH_HOME, "vendor", path) - end - - def pattern_path(path) - return ::File.join(LOGSTASH_HOME, "patterns", path) - end - def locales_path(path) return ::File.join(LOGSTASH_CORE, "locales", path) end diff --git a/lib/logstash/errors.rb b/logstash-core/lib/logstash/errors.rb similarity index 100% rename from lib/logstash/errors.rb rename to logstash-core/lib/logstash/errors.rb diff --git a/lib/logstash/filters/base.rb b/logstash-core/lib/logstash/filters/base.rb similarity index 93% rename from lib/logstash/filters/base.rb rename to logstash-core/lib/logstash/filters/base.rb index 4ce752a0e33..d2f1c601d05 100644 --- a/lib/logstash/filters/base.rb +++ b/logstash-core/lib/logstash/filters/base.rb @@ -178,12 +178,16 @@ def filter_matched(event) LogStash::Util::Decorators.add_tags(@add_tag,event,"filters/#{self.class.name}") + # note below that the tags array field needs to be updated then reassigned to the event. + # this is important because a construct like event["tags"].delete(tag) will not work + # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140 @remove_tag.each do |tag| - break if event["tags"].nil? + tags = event["tags"] + break if tags.nil? || tags.empty? tag = event.sprintf(tag) - @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag", - :tag => tag) - event["tags"].delete(tag) + @logger.debug? and @logger.debug("filters/#{self.class.name}: removing tag", :tag => tag) + tags.delete(tag) + event["tags"] = tags end end # def filter_matched diff --git a/lib/logstash/inputs/base.rb b/logstash-core/lib/logstash/inputs/base.rb similarity index 100% rename from lib/logstash/inputs/base.rb rename to logstash-core/lib/logstash/inputs/base.rb diff --git a/lib/logstash/inputs/threadable.rb b/logstash-core/lib/logstash/inputs/threadable.rb similarity index 100% rename from lib/logstash/inputs/threadable.rb rename to logstash-core/lib/logstash/inputs/threadable.rb diff --git a/lib/logstash/java_integration.rb b/logstash-core/lib/logstash/java_integration.rb similarity index 100% rename from lib/logstash/java_integration.rb rename to logstash-core/lib/logstash/java_integration.rb diff --git a/lib/logstash/json.rb b/logstash-core/lib/logstash/json.rb similarity index 100% rename from lib/logstash/json.rb rename to logstash-core/lib/logstash/json.rb diff --git a/lib/logstash/logging.rb b/logstash-core/lib/logstash/logging.rb similarity index 100% rename from lib/logstash/logging.rb rename to logstash-core/lib/logstash/logging.rb diff --git a/lib/logstash/namespace.rb b/logstash-core/lib/logstash/namespace.rb similarity index 100% rename from lib/logstash/namespace.rb rename to logstash-core/lib/logstash/namespace.rb diff --git a/lib/logstash/outputs/base.rb b/logstash-core/lib/logstash/outputs/base.rb similarity index 100% rename from lib/logstash/outputs/base.rb rename to logstash-core/lib/logstash/outputs/base.rb diff --git a/lib/logstash/patches.rb b/logstash-core/lib/logstash/patches.rb similarity index 100% rename from lib/logstash/patches.rb rename to logstash-core/lib/logstash/patches.rb diff --git a/lib/logstash/patches/bugfix_jruby_2558.rb b/logstash-core/lib/logstash/patches/bugfix_jruby_2558.rb similarity index 100% rename from lib/logstash/patches/bugfix_jruby_2558.rb rename to logstash-core/lib/logstash/patches/bugfix_jruby_2558.rb diff --git a/lib/logstash/patches/cabin.rb b/logstash-core/lib/logstash/patches/cabin.rb similarity index 100% rename from lib/logstash/patches/cabin.rb rename to logstash-core/lib/logstash/patches/cabin.rb diff --git a/lib/logstash/patches/profile_require_calls.rb b/logstash-core/lib/logstash/patches/profile_require_calls.rb similarity index 100% rename from lib/logstash/patches/profile_require_calls.rb rename to logstash-core/lib/logstash/patches/profile_require_calls.rb diff --git a/lib/logstash/patches/rubygems.rb b/logstash-core/lib/logstash/patches/rubygems.rb similarity index 100% rename from lib/logstash/patches/rubygems.rb rename to logstash-core/lib/logstash/patches/rubygems.rb diff --git a/lib/logstash/patches/stronger_openssl_defaults.rb b/logstash-core/lib/logstash/patches/stronger_openssl_defaults.rb similarity index 100% rename from lib/logstash/patches/stronger_openssl_defaults.rb rename to logstash-core/lib/logstash/patches/stronger_openssl_defaults.rb diff --git a/lib/logstash/pipeline.rb b/logstash-core/lib/logstash/pipeline.rb similarity index 100% rename from lib/logstash/pipeline.rb rename to logstash-core/lib/logstash/pipeline.rb diff --git a/lib/logstash/plugin.rb b/logstash-core/lib/logstash/plugin.rb similarity index 100% rename from lib/logstash/plugin.rb rename to logstash-core/lib/logstash/plugin.rb diff --git a/lib/logstash/program.rb b/logstash-core/lib/logstash/program.rb similarity index 100% rename from lib/logstash/program.rb rename to logstash-core/lib/logstash/program.rb diff --git a/lib/logstash/runner.rb b/logstash-core/lib/logstash/runner.rb similarity index 100% rename from lib/logstash/runner.rb rename to logstash-core/lib/logstash/runner.rb diff --git a/lib/logstash/sized_queue.rb b/logstash-core/lib/logstash/sized_queue.rb similarity index 100% rename from lib/logstash/sized_queue.rb rename to logstash-core/lib/logstash/sized_queue.rb diff --git a/lib/logstash/util.rb b/logstash-core/lib/logstash/util.rb similarity index 100% rename from lib/logstash/util.rb rename to logstash-core/lib/logstash/util.rb diff --git a/lib/logstash/util/buftok.rb b/logstash-core/lib/logstash/util/buftok.rb similarity index 100% rename from lib/logstash/util/buftok.rb rename to logstash-core/lib/logstash/util/buftok.rb diff --git a/lib/logstash/util/charset.rb b/logstash-core/lib/logstash/util/charset.rb similarity index 100% rename from lib/logstash/util/charset.rb rename to logstash-core/lib/logstash/util/charset.rb diff --git a/lib/logstash/util/decorators.rb b/logstash-core/lib/logstash/util/decorators.rb similarity index 56% rename from lib/logstash/util/decorators.rb rename to logstash-core/lib/logstash/util/decorators.rb index 0ea2c021aca..265656e5ce9 100644 --- a/lib/logstash/util/decorators.rb +++ b/logstash-core/lib/logstash/util/decorators.rb @@ -19,13 +19,16 @@ def add_fields(fields,event, pluginname) value.each do |v| v = event.sprintf(v) if event.include?(field) - event[field] = Array(event[field]) - event[field] << v + # note below that the array field needs to be updated then reassigned to the event. + # this is important because a construct like event[field] << v will not work + # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140 + a = Array(event[field]) + a << v + event[field] = a else event[field] = v end - @logger.debug? and @logger.debug("#{pluginname}: adding value to field", - :field => field, :value => value) + @logger.debug? and @logger.debug("#{pluginname}: adding value to field", :field => field, :value => value) end end end @@ -34,9 +37,13 @@ def add_fields(fields,event, pluginname) def add_tags(tags, event, pluginname) tags.each do |tag| tag = event.sprintf(tag) - @logger.debug? and @logger.debug("#{pluginname}: adding tag", - :tag => tag) - (event["tags"] ||= []) << tag + @logger.debug? and @logger.debug("#{pluginname}: adding tag", :tag => tag) + # note below that the tags array field needs to be updated then reassigned to the event. + # this is important because a construct like event["tags"] << tag will not work + # in the current Java event implementation. see https://github.com/elastic/logstash/issues/4140 + tags = event["tags"] || [] + tags << tag + event["tags"] = tags end end diff --git a/lib/logstash/util/defaults_printer.rb b/logstash-core/lib/logstash/util/defaults_printer.rb similarity index 100% rename from lib/logstash/util/defaults_printer.rb rename to logstash-core/lib/logstash/util/defaults_printer.rb diff --git a/lib/logstash/util/filetools.rb b/logstash-core/lib/logstash/util/filetools.rb similarity index 100% rename from lib/logstash/util/filetools.rb rename to logstash-core/lib/logstash/util/filetools.rb diff --git a/lib/logstash/util/java_version.rb b/logstash-core/lib/logstash/util/java_version.rb similarity index 100% rename from lib/logstash/util/java_version.rb rename to logstash-core/lib/logstash/util/java_version.rb diff --git a/lib/logstash/util/password.rb b/logstash-core/lib/logstash/util/password.rb similarity index 100% rename from lib/logstash/util/password.rb rename to logstash-core/lib/logstash/util/password.rb diff --git a/lib/logstash/util/plugin_version.rb b/logstash-core/lib/logstash/util/plugin_version.rb similarity index 100% rename from lib/logstash/util/plugin_version.rb rename to logstash-core/lib/logstash/util/plugin_version.rb diff --git a/lib/logstash/util/prctl.rb b/logstash-core/lib/logstash/util/prctl.rb similarity index 100% rename from lib/logstash/util/prctl.rb rename to logstash-core/lib/logstash/util/prctl.rb diff --git a/lib/logstash/util/reporter.rb b/logstash-core/lib/logstash/util/reporter.rb similarity index 100% rename from lib/logstash/util/reporter.rb rename to logstash-core/lib/logstash/util/reporter.rb diff --git a/lib/logstash/util/retryable.rb b/logstash-core/lib/logstash/util/retryable.rb similarity index 100% rename from lib/logstash/util/retryable.rb rename to logstash-core/lib/logstash/util/retryable.rb diff --git a/lib/logstash/util/socket_peer.rb b/logstash-core/lib/logstash/util/socket_peer.rb similarity index 100% rename from lib/logstash/util/socket_peer.rb rename to logstash-core/lib/logstash/util/socket_peer.rb diff --git a/lib/logstash/util/unicode_trimmer.rb b/logstash-core/lib/logstash/util/unicode_trimmer.rb similarity index 100% rename from lib/logstash/util/unicode_trimmer.rb rename to logstash-core/lib/logstash/util/unicode_trimmer.rb diff --git a/lib/logstash/util/worker_threads_default_printer.rb b/logstash-core/lib/logstash/util/worker_threads_default_printer.rb similarity index 100% rename from lib/logstash/util/worker_threads_default_printer.rb rename to logstash-core/lib/logstash/util/worker_threads_default_printer.rb diff --git a/logstash-core/lib/logstash/version.rb b/logstash-core/lib/logstash/version.rb new file mode 100644 index 00000000000..70715b097cb --- /dev/null +++ b/logstash-core/lib/logstash/version.rb @@ -0,0 +1,14 @@ +# encoding: utf-8 + +# The version of the logstash package (not the logstash-core gem version). +# +# Note to authors: this should not include dashes because 'gem' barfs if +# you include a dash in the version string. + +# TODO: (colin) the logstash-core gem uses it's own version number in logstash-core/lib/logstash-core/version.rb +# there are some dependencies in logstash-core on the LOGSTASH_VERSION constant this is why +# the logstash version is currently defined here in logstash-core/lib/logstash/version.rb but +# eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be +# fixed. + +LOGSTASH_VERSION = "3.0.0.dev" diff --git a/locales/en.yml b/logstash-core/locales/en.yml similarity index 100% rename from locales/en.yml rename to logstash-core/locales/en.yml diff --git a/logstash-core.gemspec b/logstash-core/logstash-core.gemspec similarity index 90% rename from logstash-core.gemspec rename to logstash-core/logstash-core.gemspec index 35937a060a1..595c5f413c0 100644 --- a/logstash-core.gemspec +++ b/logstash-core/logstash-core.gemspec @@ -1,7 +1,7 @@ # -*- encoding: utf-8 -*- lib = File.expand_path('../lib', __FILE__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require 'logstash/version' +require 'logstash-core/version' Gem::Specification.new do |gem| gem.authors = ["Jordan Sissel", "Pete Fritchman", "Elasticsearch"] @@ -11,11 +11,13 @@ Gem::Specification.new do |gem| gem.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" gem.license = "Apache License (2.0)" - gem.files = Dir.glob(["logstash-core.gemspec", "lib/logstash-core.rb", "lib/logstash/**/*.rb", "spec/**/*.rb", "locales/*"]) + gem.files = Dir.glob(["logstash-core.gemspec", "lib/**/*.rb", "spec/**/*.rb", "locales/*"]) gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) gem.name = "logstash-core" gem.require_paths = ["lib"] - gem.version = LOGSTASH_VERSION + gem.version = LOGSTASH_CORE_VERSION + + gem.add_runtime_dependency "logstash-core-event", "~> 3.0.0.dev" gem.add_runtime_dependency "cabin", "~> 0.7.0" #(Apache 2.0 license) gem.add_runtime_dependency "pry", "~> 0.10.1" #(Ruby license) diff --git a/spec/core/conditionals_spec.rb b/logstash-core/spec/conditionals_spec.rb similarity index 100% rename from spec/core/conditionals_spec.rb rename to logstash-core/spec/conditionals_spec.rb diff --git a/spec/logstash/agent_spec.rb b/logstash-core/spec/logstash/agent_spec.rb similarity index 100% rename from spec/logstash/agent_spec.rb rename to logstash-core/spec/logstash/agent_spec.rb diff --git a/spec/core/config_spec.rb b/logstash-core/spec/logstash/config/config_ast_spec.rb similarity index 100% rename from spec/core/config_spec.rb rename to logstash-core/spec/logstash/config/config_ast_spec.rb diff --git a/spec/core/config_cpu_core_strategy_spec.rb b/logstash-core/spec/logstash/config/cpu_core_strategy_spec.rb similarity index 100% rename from spec/core/config_cpu_core_strategy_spec.rb rename to logstash-core/spec/logstash/config/cpu_core_strategy_spec.rb diff --git a/spec/core/config_defaults_spec.rb b/logstash-core/spec/logstash/config/defaults_spec.rb similarity index 100% rename from spec/core/config_defaults_spec.rb rename to logstash-core/spec/logstash/config/defaults_spec.rb diff --git a/spec/core/config_mixin_spec.rb b/logstash-core/spec/logstash/config/mixin_spec.rb similarity index 100% rename from spec/core/config_mixin_spec.rb rename to logstash-core/spec/logstash/config/mixin_spec.rb diff --git a/spec/core/environment_spec.rb b/logstash-core/spec/logstash/environment_spec.rb similarity index 100% rename from spec/core/environment_spec.rb rename to logstash-core/spec/logstash/environment_spec.rb diff --git a/spec/filters/base_spec.rb b/logstash-core/spec/logstash/filters/base_spec.rb similarity index 100% rename from spec/filters/base_spec.rb rename to logstash-core/spec/logstash/filters/base_spec.rb diff --git a/spec/inputs/base_spec.rb b/logstash-core/spec/logstash/inputs/base_spec.rb similarity index 100% rename from spec/inputs/base_spec.rb rename to logstash-core/spec/logstash/inputs/base_spec.rb diff --git a/spec/lib/logstash/java_integration_spec.rb b/logstash-core/spec/logstash/java_integration_spec.rb similarity index 100% rename from spec/lib/logstash/java_integration_spec.rb rename to logstash-core/spec/logstash/java_integration_spec.rb diff --git a/spec/util/json_spec.rb b/logstash-core/spec/logstash/json_spec.rb similarity index 100% rename from spec/util/json_spec.rb rename to logstash-core/spec/logstash/json_spec.rb diff --git a/spec/outputs/base_spec.rb b/logstash-core/spec/logstash/outputs/base_spec.rb similarity index 100% rename from spec/outputs/base_spec.rb rename to logstash-core/spec/logstash/outputs/base_spec.rb diff --git a/spec/logstash/patches_spec.rb b/logstash-core/spec/logstash/patches_spec.rb similarity index 100% rename from spec/logstash/patches_spec.rb rename to logstash-core/spec/logstash/patches_spec.rb diff --git a/spec/core/pipeline_spec.rb b/logstash-core/spec/logstash/pipeline_spec.rb similarity index 100% rename from spec/core/pipeline_spec.rb rename to logstash-core/spec/logstash/pipeline_spec.rb diff --git a/spec/core/plugin_spec.rb b/logstash-core/spec/logstash/plugin_spec.rb similarity index 100% rename from spec/core/plugin_spec.rb rename to logstash-core/spec/logstash/plugin_spec.rb diff --git a/spec/core/runner_spec.rb b/logstash-core/spec/logstash/runner_spec.rb similarity index 100% rename from spec/core/runner_spec.rb rename to logstash-core/spec/logstash/runner_spec.rb diff --git a/spec/util/buftok_spec.rb b/logstash-core/spec/logstash/util/buftok_spec.rb similarity index 100% rename from spec/util/buftok_spec.rb rename to logstash-core/spec/logstash/util/buftok_spec.rb diff --git a/spec/util/charset_spec.rb b/logstash-core/spec/logstash/util/charset_spec.rb similarity index 100% rename from spec/util/charset_spec.rb rename to logstash-core/spec/logstash/util/charset_spec.rb diff --git a/spec/util/defaults_printer_spec.rb b/logstash-core/spec/logstash/util/defaults_printer_spec.rb similarity index 100% rename from spec/util/defaults_printer_spec.rb rename to logstash-core/spec/logstash/util/defaults_printer_spec.rb diff --git a/spec/util/java_version_spec.rb b/logstash-core/spec/logstash/util/java_version_spec.rb similarity index 100% rename from spec/util/java_version_spec.rb rename to logstash-core/spec/logstash/util/java_version_spec.rb diff --git a/spec/util/plugin_version_spec.rb b/logstash-core/spec/logstash/util/plugin_version_spec.rb similarity index 100% rename from spec/util/plugin_version_spec.rb rename to logstash-core/spec/logstash/util/plugin_version_spec.rb diff --git a/spec/util/unicode_trimmer_spec.rb b/logstash-core/spec/logstash/util/unicode_trimmer_spec.rb similarity index 100% rename from spec/util/unicode_trimmer_spec.rb rename to logstash-core/spec/logstash/util/unicode_trimmer_spec.rb diff --git a/spec/util/worker_threads_default_printer_spec.rb b/logstash-core/spec/logstash/util/worker_threads_default_printer_spec.rb similarity index 100% rename from spec/util/worker_threads_default_printer_spec.rb rename to logstash-core/spec/logstash/util/worker_threads_default_printer_spec.rb diff --git a/spec/util_spec.rb b/logstash-core/spec/logstash/util_spec.rb similarity index 100% rename from spec/util_spec.rb rename to logstash-core/spec/logstash/util_spec.rb diff --git a/logstash-event.gemspec b/logstash-event.gemspec deleted file mode 100644 index ea6cce87e1a..00000000000 --- a/logstash-event.gemspec +++ /dev/null @@ -1,41 +0,0 @@ -# -*- encoding: utf-8 -*- -Gem::Specification.new do |gem| - gem.authors = ["Jordan Sissel"] - gem.email = ["jls@semicomplete.com"] - gem.description = %q{Library that contains the classes required to create LogStash events} - gem.summary = %q{Library that contains the classes required to create LogStash events} - gem.homepage = "https://github.com/logstash/logstash" - gem.license = "Apache License (2.0)" - - gem.files = %w{ - lib/logstash-event.rb - lib/logstash/environment.rb - lib/logstash/errors.rb - lib/logstash/event.rb - lib/logstash/java_integration.rb - lib/logstash/json.rb - lib/logstash/namespace.rb - lib/logstash/timestamp.rb - lib/logstash/version.rb - lib/logstash/util.rb - lib/logstash/util/accessors.rb - LICENSE - } - - gem.test_files = ["spec/core/event_spec.rb"] - gem.name = "logstash-event" - gem.require_paths = ["lib"] - gem.version = "1.3.0" - - gem.add_runtime_dependency "cabin" - gem.add_development_dependency "rspec" - gem.add_development_dependency "guard" - gem.add_development_dependency "guard-rspec" - - if RUBY_PLATFORM == 'java' - gem.platform = RUBY_PLATFORM - gem.add_runtime_dependency "jrjackson" - else - gem.add_runtime_dependency "oj" - end -end diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index e2637f09995..fd66801588f 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -1,4 +1,4 @@ -require "logstash/version" +require "logstash-core/lib/logstash/version" namespace "artifact" do @@ -54,20 +54,46 @@ namespace "artifact" do File.open(".bundle/config", "w") { } end - # locate the "gem "logstash-core" ..." line in Gemfile, and if the :path => "." option if specified + # locate the "gem "logstash-core" ..." line in Gemfile, and if the :path => "..." option if specified # build and install the local logstash-core gem otherwise just do nothing, bundler will deal with it. task "install-logstash-core" do + # regex which matches a Gemfile gem definition for the logstash-core gem and captures the :path option + gem_line_regex = /^\s*gem\s+["']logstash-core["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i + lines = File.readlines("Gemfile") - matches = lines.select{|line| line[/^gem\s+["']logstash-core["']/i]} + matches = lines.select{|line| line[gem_line_regex]} abort("ERROR: Gemfile format error, need a single logstash-core gem specification") if matches.size != 1 - if matches.first =~ /:path\s*=>\s*["']\.["']/ - Rake::Task["plugin:install-local-logstash-core-gem"].invoke + + path = matches.first[gem_line_regex, 1] + + if path + Rake::Task["plugin:install-local-core-gem"].invoke("logstash-core", path) else puts("[artifact:install-logstash-core] using logstash-core from Rubygems") end end - task "prepare" => ["bootstrap", "plugin:install-default", "install-logstash-core", "clean-bundle-config"] + # # locate the "gem "logstash-core-event*" ..." line in Gemfile, and if the :path => "." option if specified + # # build and install the local logstash-core-event* gem otherwise just do nothing, bundler will deal with it. + task "install-logstash-core-event" do + # regex which matches a Gemfile gem definition for the logstash-core-event* gem and captures the gem name and :path option + gem_line_regex = /^\s*gem\s+["'](logstash-core-event[^"^']*)["'](?:\s*,\s*["'][^"^']+["'])?(?:\s*,\s*:path\s*=>\s*["']([^"^']+)["'])?/i + + lines = File.readlines("Gemfile") + matches = lines.select{|line| line[gem_line_regex]} + abort("ERROR: Gemfile format error, need a single logstash-core-event gem specification") if matches.size != 1 + + name = matches.first[gem_line_regex, 1] + path = matches.first[gem_line_regex, 2] + + if path + Rake::Task["plugin:install-local-core-gem"].invoke(name, path) + else + puts("[artifact:install-logstash-core] using #{name} from Rubygems") + end + end + + task "prepare" => ["bootstrap", "plugin:install-default", "install-logstash-core", "install-logstash-core-event", "clean-bundle-config"] desc "Build a tar.gz of logstash with all dependencies" task "tar" => ["prepare"] do diff --git a/rakelib/compile.rake b/rakelib/compile.rake index 7076fbf3ca9..be5693bead7 100644 --- a/rakelib/compile.rake +++ b/rakelib/compile.rake @@ -8,12 +8,15 @@ end namespace "compile" do desc "Compile the config grammar" - task "grammar" => "lib/logstash/config/grammar.rb" + + task "grammar" => "logstash-core/lib/logstash/config/grammar.rb" desc "Build everything" - task "all" => "grammar" + # task "all" => ["grammar", "logstash-core-event-java"] + task "all" => ["grammar"] - task "jruby-event" do - sh './java/gradlew logstash-event:shadowJar -p ./java' + task "logstash-core-event-java" do + puts("Building logstash-core-event-java using gradle") + system("logstash-core-event-java/gradlew", "jar", "-p", "./logstash-core-event-java") end end diff --git a/rakelib/plugin.rake b/rakelib/plugin.rake index 9c2065c1f56..d08fbdf6f2a 100644 --- a/rakelib/plugin.rake +++ b/rakelib/plugin.rake @@ -57,26 +57,40 @@ namespace "plugin" do task.reenable # Allow this task to be run again end - task "clean-logstash-core-gem" do - Dir["logstash-core*.gem"].each do |gem| + task "clean-local-core-gem", [:name, :path] do |task, args| + name = args[:name] + path = args[:path] + + Dir[File.join(path, "#{name}*.gem")].each do |gem| + puts("[plugin:clean-local-core-gem] Cleaning #{gem}") rm(gem) end task.reenable # Allow this task to be run again end - task "build-logstash-core-gem" => [ "clean-logstash-core-gem" ] do - puts("[plugin:build-logstash-core-gem] Building logstash-core.gemspec") + task "build-local-core-gem", [:name, :path] do |task, args| + name = args[:name] + path = args[:path] + + Rake::Task["plugin:clean-local-core-gem"].invoke(name, path) - system("gem build logstash-core.gemspec") + puts("[plugin:build-local-core-gem] Building #{File.join(path, name)}.gemspec") + + system("cd #{path}; gem build #{name}.gemspec") task.reenable # Allow this task to be run again end - task "install-local-logstash-core-gem" => [ "build-logstash-core-gem" ] do - gems = Dir["logstash-core*.gem"] - abort("ERROR: logstash-core gem not found") if gems.size != 1 - puts("[plugin:install-local-logstash-core-gem] Installing #{gems.first}") + task "install-local-core-gem", [:name, :path] do |task, args| + name = args[:name] + path = args[:path] + + Rake::Task["plugin:build-local-core-gem"].invoke(name, path) + + gems = Dir[File.join(path, "#{name}*.gem")] + abort("ERROR: #{name} gem not found in #{path}") if gems.size != 1 + puts("[plugin:install-local-core-gem] Installing #{gems.first}") install_plugins("--no-verify", gems.first) task.reenable # Allow this task to be run again diff --git a/rakelib/test.rake b/rakelib/test.rake index 7ac22c304af..8c0d16ff4ef 100644 --- a/rakelib/test.rake +++ b/rakelib/test.rake @@ -19,18 +19,36 @@ namespace "test" do require 'ci/reporter/rake/rspec_loader' end + def core_specs + # note that regardless if which logstash-core-event-* gem is live, we will always run the + # logstash-core-event specs since currently this is the most complete Event and Timestamp specs + # which actually defines the Event contract and should pass regardless of the actuall underlying + # implementation. + specs = ["spec/**/*_spec.rb", "logstash-core/spec/**/*_spec.rb", "logstash-core-event/spec/**/*_spec.rb"] + + # figure if the logstash-core-event-java gem is loaded and if so add its specific specs in the core specs to run + begin + require "logstash-core-event-java/version" + specs << "logstash-core-event-java/spec/**/*_spec.rb" + rescue LoadError + # logstash-core-event-java gem is not live, ignore and skip specs + end + + Rake::FileList[*specs] + end + desc "run core specs" task "core" => ["setup"] do - exit(RSpec::Core::Runner.run([Rake::FileList["spec/**/*_spec.rb"]])) + exit(RSpec::Core::Runner.run([core_specs])) end desc "run core specs in fail-fast mode" task "core-fail-fast" => ["setup"] do - exit(RSpec::Core::Runner.run(["--fail-fast", Rake::FileList["spec/**/*_spec.rb"]])) + exit(RSpec::Core::Runner.run(["--fail-fast", core_specs])) end desc "run core specs on a single file" - task "core-single-file", [:specfile] => ["setup"] do |t,args| + task "core-single-file", [:specfile] => ["setup"] do |t, args| exit(RSpec::Core::Runner.run([Rake::FileList[args.specfile]])) end diff --git a/require-analyze.rb b/require-analyze.rb deleted file mode 100644 index f69d858aa45..00000000000 --- a/require-analyze.rb +++ /dev/null @@ -1,22 +0,0 @@ -require "csv" - -#0.003,psych/nodes/mapping,/Users/jls/.rvm/rubies/jruby-1.7.8/lib/ruby/shared/psych/nodes.rb:6:in `(root)' - -durations = {} -durations.default = 0 - -CSV.foreach(ARGV[0]) do |duration, path, source| - source, line, where = source.split(":") - #{"0.002"=>"/Users/jls/projects/logstash/vendor/bundle/jruby/1.9/gems/clamp-0.6.3/lib/clamp.rb"} - if source.include?("jruby/1.9/gems") - # Get the gem name - source = source.gsub(/.*\/jruby\/1.9\/gems/, "")[/[^\/]+/] - elsif source.include?("/lib/logstash/") - source = source.gsub(/^.*(\/lib\/logstash\/)/, "/lib/logstash/") - end - durations[source] += duration.to_f -end - -durations.sort_by { |k,v| v }.each do |k,v| - puts "#{v} #{k}" -end diff --git a/spec/lib/logstash/bundler_spec.rb b/spec/bootstrap/bundler_spec.rb similarity index 100% rename from spec/lib/logstash/bundler_spec.rb rename to spec/bootstrap/bundler_spec.rb diff --git a/spec/util/gemfile_spec.rb b/spec/pluginmanager/gemfile_spec.rb similarity index 100% rename from spec/util/gemfile_spec.rb rename to spec/pluginmanager/gemfile_spec.rb