Skip to content

Commit

Permalink
Release 2.1.0
Browse files Browse the repository at this point in the history
  • Loading branch information
wuchong committed Nov 12, 2015
1 parent e935da9 commit d56ca39
Show file tree
Hide file tree
Showing 1,085 changed files with 61,824 additions and 48,582 deletions.
10 changes: 10 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
*.git
*.project
*.classpath
*.prefs
*.bat
target/*
*/target/*
/bin
.idea
*.iml
37 changes: 28 additions & 9 deletions bin/jstorm.py
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,9 @@ def get_client_childopts():
return ret

def get_server_childopts(log_name):
ret = (" -Dlogfile.name=%s -Dlogback.configurationFile=%s" %(log_name, LOGBACK_CONF))
jstorm_log_dir = get_log_dir()
gc_log_path = jstorm_log_dir + "/" + log_name + ".gc"
ret = (" -Xloggc:%s -Dlogfile.name=%s -Dlogback.configurationFile=%s -Djstorm.log.dir=%s " %(gc_log_path, log_name, LOGBACK_CONF, jstorm_log_dir))
return ret

if not os.path.exists(JSTORM_DIR + "/RELEASE"):
Expand Down Expand Up @@ -103,8 +105,8 @@ def get_classpath(extrajars):
ret.extend(get_jars_full(JSTORM_DIR))
ret.extend(get_jars_full(JSTORM_DIR + "/lib"))
ret.extend(INCLUDE_JARS)

return normclasspath(":".join(ret))


def confvalue(name, extrapaths):
command = [
Expand All @@ -130,6 +132,18 @@ def print_localconfvalue(name):
"""
print name + ": " + confvalue(name, [JSTORM_CONF_DIR])

def get_log_dir():
cppaths = [JSTORM_CONF_DIR]
jstorm_log_dir = confvalue("jstorm.log.dir", cppaths)
if not jstorm_log_dir == "null":
if not os.path.exists(jstorm_log_dir):
os.mkdir(jstorm_log_dir)
else:
jstorm_log_dir = JSTORM_DIR + "/logs"
if not os.path.exists(jstorm_log_dir):
os.mkdir(jstorm_log_dir)
return jstorm_log_dir

def print_remoteconfvalue(name):
"""Syntax: [jstorm remoteconfvalue conf-name]
Expand All @@ -141,11 +155,17 @@ def print_remoteconfvalue(name):
"""
print name + ": " + confvalue(name, [JSTORM_CONF_DIR])


def exec_storm_class(klass, jvmtype="-server", childopts="", extrajars=[], args=[]):
nativepath = confvalue("java.library.path", extrajars)
args_str = " ".join(map(lambda s: "\"" + s + "\"", args))
command = "java " + jvmtype + " -Djstorm.home=" + JSTORM_DIR + " " + get_config_opts() + " -Djava.library.path=" + nativepath + " " + childopts + " -cp " + get_classpath(extrajars) + " " + klass + " " + args_str
print "Running: " + command
print args_str
if "NimbusServer" in klass:
# fix cmd > 4096, use dir in cp, only for nimbus server
command = "java " + jvmtype + " -Djstorm.home=" + JSTORM_DIR + " " + get_config_opts() + " -Djava.library.path=" + nativepath + " " + childopts + " -cp " + get_classpath(extrajars) + ":" + JSTORM_DIR + "/lib/ext/* " + klass + " " + args_str
else:
command = "java " + jvmtype + " -Djstorm.home=" + JSTORM_DIR + " " + get_config_opts() + " -Djava.library.path=" + nativepath + " " + childopts + " -cp " + get_classpath(extrajars) + " " + klass + " " + args_str
print "Running: " + command
global STATUS
STATUS = os.system(command)

Expand Down Expand Up @@ -263,12 +283,12 @@ def restart(*args):
extrajars=[JSTORM_CONF_DIR, JSTORM_DIR + "/bin", CLIENT_CONF_FILE],
childopts=childopts)

def update_config(*args):
"""Syntax: [jstorm restart topology-name [conf]]
def update_topology(*args):
"""Syntax: [jstorm update_topology topology-name -jar [jarpath] -conf [confpath]]
"""
childopts = get_client_childopts()
exec_storm_class(
"backtype.storm.command.update_config",
"backtype.storm.command.update_topology",
args=args,
jvmtype="-client -Xms256m -Xmx256m",
extrajars=[JSTORM_CONF_DIR, JSTORM_DIR + "/bin", CLIENT_CONF_FILE],
Expand Down Expand Up @@ -309,7 +329,6 @@ def supervisor():
extrajars=cppaths,
childopts=childopts)


def drpc():
"""Syntax: [jstorm drpc]
Expand Down Expand Up @@ -388,7 +407,7 @@ def list(*args):
"drpc": drpc, "supervisor": supervisor, "localconfvalue": print_localconfvalue,
"remoteconfvalue": print_remoteconfvalue, "classpath": print_classpath,
"activate": activate, "deactivate": deactivate, "rebalance": rebalance, "help": print_usage,
"metricsMonitor": metrics_Monitor, "list": list, "restart": restart, "update_config": update_config}
"metricsMonitor": metrics_Monitor, "list": list, "restart": restart, "update_topology": update_topology}

def parse_config(config_list):
global CONFIG_OPTS
Expand Down
16 changes: 9 additions & 7 deletions bin/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ else
fi
echo "JSTORM_HOME =" $JSTORM_HOME

if [ "x$JSTORM_CONF_DIR_PATH" != "x" ]
if [ "x$JSTORM_CONF_DIR" != "x" ]
then
echo "JSTORM_CONF_DIR_PATH has been set "
echo "JSTORM_CONF_DIR has been set "
else
export JSTORM_CONF_DIR_PATH=$JSTORM_HOME/conf
export JSTORM_CONF_DIR=$JSTORM_HOME/conf
fi
echo "JSTORM_CONF_DIR_PATH =" $JSTORM_CONF_DIR_PATH
echo "JSTORM_CONF_DIR =" $JSTORM_CONF_DIR



Expand Down Expand Up @@ -62,16 +62,18 @@ function startJStorm()


HOSTNAME=`hostname -i`
NIMBUS_HOST=`grep "nimbus.host:" $JSTORM_CONF_DIR_PATH/storm.yaml | grep -w $HOSTNAME`
SUPERVISOR_HOST_START=`grep "supervisor.host.start:" $JSTORM_CONF_DIR_PATH/storm.yaml | grep -w "false"`
NIMBUS_HOST=`grep "nimbus.host:" $JSTORM_CONF_DIR/storm.yaml |grep -v "#" | grep -w $HOSTNAME`
NIMBUS_HOST_START_SUPERVISOR=`grep "nimbus.host.start.supervisor:" $JSTORM_CONF_DIR/storm.yaml |grep -v "#" | grep -wi "false"`

if [ "X${NIMBUS_HOST}" != "X" ]
then
startJStorm "nimbus" "NimbusServer"
fi

if [ "X${SUPERVISOR_HOST_START}" == "X" ]
if [ "X${NIMBUS_HOST}" != "X" ] && [ "X${NIMBUS_HOST_START_SUPERVISOR}" != "X" ]
then
echo "Skip start Supervisor on nimbus host"
else
startJStorm "supervisor" "Supervisor"
fi

Expand Down
Empty file modified bin/stop.sh
100755 → 100644
Empty file.
Empty file modified conf/cgconfig.conf
100755 → 100644
Empty file.
Empty file modified conf/client_log4j.properties
100755 → 100644
Empty file.
7 changes: 4 additions & 3 deletions conf/client_logback.xml
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@
</encoder>
</appender>
<logger name="org.apache.zookeeper" level="ERROR" />
<logger name="org.apache.curator" level="ERROR" />
<logger name="com.netflix.curator" level="ERROR" />
<logger name="com.alibaba.jstorm.common.metric" level="ERROR" />
<logger name="com.alibaba.jstorm.daemon.nimbus.TopologyMetricsRunnable" level="ERROR" />
<logger name="com.alibaba.jstorm.metric" level="ERROR" />
<logger name="com.alibaba.jstorm.common.metric" level="INFO" />
<logger name="com.alibaba.jstorm.daemon.nimbus.TopologyMetricsRunnable" level="INFO" />
<logger name="com.alibaba.jstorm.metric" level="INFO" />

<root level="INFO">
<appender-ref ref="A1" />
Expand Down
4 changes: 2 additions & 2 deletions conf/jstorm.log4j.properties
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ log4j.appender.stdout.layout.ConversionPattern = [%p %d{yyyy-MM-dd HH:mm:ss} %

### output to file ###
log4j.appender.D = org.apache.log4j.RollingFileAppender
log4j.appender.D.File = ${jstorm.home}/logs/${logfile.name}
log4j.appender.D.File = ${jstorm.log.dir}/${logfile.name}
log4j.appender.D.Append = true
log4j.appender.D.Threshold = INFO
log4j.appender.D.MaxFileSize=1GB
Expand All @@ -37,7 +37,7 @@ log4j.logger.com.alibaba.jstorm.metric= INFO, M
log4j.additivity.com.alibaba.jstorm.metric=false

log4j.appender.M = org.apache.log4j.RollingFileAppender
log4j.appender.M.File = ${jstorm.home}/logs/${logfile.name}.metrics
log4j.appender.M.File = ${jstorm.log.dir}/${logfile.name}.metrics
log4j.appender.M.Append = true
log4j.appender.M.Threshold = INFO
log4j.appender.M.MaxFileSize=100MB
Expand Down
130 changes: 61 additions & 69 deletions conf/jstorm.logback.xml
100755 → 100644
Original file line number Diff line number Diff line change
@@ -1,84 +1,76 @@
<?xml version="1.0" encoding="UTF-8"?>

<configuration scan="true" scanPeriod="60 seconds">
<appender name="A1"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${jstorm.home}/logs/${logfile.name}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>${jstorm.home}/logs/${logfile.name}.%i</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>5</maxIndex>
</rollingPolicy>
<appender name="A1" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${jstorm.log.dir}/${topology.name:-.}/${logfile.name}</file>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>${jstorm.log.dir}/${topology.name:-.}/${logfile.name}.%i</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>5</maxIndex>
</rollingPolicy>

<triggeringPolicy
class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>1GB</maxFileSize>
</triggeringPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>1GB</maxFileSize>
</triggeringPolicy>

<encoder>
<pattern>[%p %d{yyyy-MM-dd HH:mm:ss} %c{1}:%L %t] %m%n</pattern>
<encoder>
<pattern>[%p %d{yyyy-MM-dd HH:mm:ss} %c{1}:%L %t] %m%n</pattern>
</encoder>
</appender>

</encoder>
</appender>
<appender name="METRICS" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${jstorm.log.dir}/${topology.name:-.}/${logfile.name}.metrics</file>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>${jstorm.log.dir}/${topology.name:-.}/${logfile.name}.metrics.%i</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>5</maxIndex>
</rollingPolicy>

<appender name="METRICS"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${jstorm.home}/logs/${logfile.name}.metrics</file>
<rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
<fileNamePattern>${jstorm.home}/logs/${logfile.name}.metrics.%i</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>5</maxIndex>
</rollingPolicy>
<triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>100MB</maxFileSize>
</triggeringPolicy>

<triggeringPolicy
class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
<maxFileSize>100MB</maxFileSize>
</triggeringPolicy>
<encoder>
<!--<pattern>[%p %d{yyyy-MM-dd HH:mm:ss} %c{1}:%L %t] %m%n</pattern>-->
<pattern>[%d{yyyy-MM-dd HH:mm:ss} %t] %m%n</pattern>
</encoder>
</appender>

<encoder>
<pattern>[%p %d{yyyy-MM-dd HH:mm:ss} %c{1}:%L %t] %m%n</pattern>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="A1"/>
</root>
<logger name="org.apache.zookeeper" level="ERROR" />
<logger name="org.apache.curator" level="ERROR" />
<logger name="com.netflix.curator" level="ERROR" />

<root level="INFO">
<appender-ref ref="A1" />
</root>
<logger name="com.alibaba.jstorm" additivity="false">
<level value="INFO"/>
<appender-ref ref="A1"/>
</logger>

<logger name="com.alibaba.jstorm"
additivity="false">
<level value="INFO" />
<appender-ref ref="A1" />
</logger>
<logger name="com.alibaba.jstorm.common.metric" additivity="false">
<level value="INFO"/>
<appender-ref ref="METRICS"/>
</logger>

<logger name="com.alibaba.jstorm.common.metric"
additivity="false">
<level value="INFO" />
<appender-ref ref="METRICS" />
</logger>

<logger name="com.alibaba.jstorm.task.heartbeat"
additivity="false">
<level value="INFO" />
<appender-ref ref="METRICS" />
</logger>

<logger name="com.alibaba.jstorm.daemon.worker.hearbeat"
additivity="false">
<level value="INFO" />
<appender-ref ref="METRICS" />
</logger>

<logger name="com.alibaba.jstorm.daemon.nimbus.TopologyMetricsRunnable"
additivity="false">
<level value="INFO" />
<appender-ref ref="METRICS" />
</logger>

<logger name="com.alibaba.jstorm.metric"
additivity="false">
<level value="INFO" />
<appender-ref ref="METRICS" />
</logger>
<logger name="com.alibaba.jstorm.task.heartbeat" additivity="false">
<level value="WARN"/>
<appender-ref ref="METRICS"/>
</logger>

<logger name="com.alibaba.jstorm.daemon.worker.hearbeat" additivity="false">
<level value="WARN"/>
<appender-ref ref="METRICS"/>
</logger>

<logger name="com.alibaba.jstorm.daemon.nimbus.metric" additivity="false">
<level value="INFO"/>
<appender-ref ref="METRICS"/>
</logger>

<logger name="com.alibaba.jstorm.metric" additivity="false">
<level value="INFO"/>
<appender-ref ref="METRICS"/>
</logger>
</configuration>

Loading

0 comments on commit d56ca39

Please sign in to comment.