Skip to content

Commit c838a6f

Browse files
bcarrierbcarrier
committed
Merge pull request #2 from ihelmke/master
Cloud/EC2 Changes
2 parents d8c5f8a + c3bc77d commit c838a6f

File tree

5 files changed

+132
-0
lines changed

5 files changed

+132
-0
lines changed

‎deploy/setup.sh

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
sudo apt-get install unzip
2+
sudo unzip -n -qq sleuthkit-pipeline-1-SNAPSHOT-libs.zip -d /usr/lib/hadoop/lib/
3+
for SERVICE in /etc/init.d/hadoop*
4+
do
5+
sudo $SERVICE restart
6+
done
7+
echo "export LD_LIBRARY_PATH=$HOME/txpete/fsrip/deps/" >> $HOME/.bashrc
8+
echo "export PATH=$PATH:$HOME/txpete/fsrip/" >> $HOME/.bashrc
9+
echo "export HADOO_HOME=/usr/lib/hadoop/" >> $HOME/.bashrc

‎deploy/tpkickoff.sh

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/sh
2+
3+
if [ $# -ne 3 ] && [ $# -ne 4 ]
4+
then
5+
echo "Usage: tpkickoff.sh image_friendly_name image_path jar_dir"
6+
exit 1
7+
fi
8+
9+
if [ $# -eq 4 ]
10+
then
11+
if [ ${#4} -ne 32 ]
12+
then
13+
echo "Invalid Image ID: Must be 32 digits long"
14+
exit 1
15+
fi
16+
fi
17+
18+
pwd
19+
date
20+
21+
FriendlyName=$1
22+
ImagePath=$2
23+
JarDir=$3
24+
25+
JarFile=`ls $JarDir/sleuthkit-pipeline-r*-job.jar | sort | tail -n 1`
26+
27+
JsonFile=$FriendlyName.json
28+
HdfsImage=$FriendlyName.dd
29+
30+
echo "jar file is ${JarFile}"
31+
32+
# rip filesystem metadata, upload to hdfs
33+
fsrip dumpfs $ImagePath | $HADOOP_HOME/bin/hadoop jar $JarFile com.lightboxtechnologies.spectrum.Uploader $JsonFile
34+
35+
# upload image to hdfs
36+
ImageID=`cat $ImagePath | $HADOOP_HOME/bin/hadoop jar $JarFile com.lightboxtechnologies.spectrum.Uploader $HdfsImage`
37+
echo "done uploading"
38+
if [ $# -eq 4 ]
39+
then
40+
ImageID=$4
41+
fi
42+
echo "Image ID is ${ImageID}"
43+
44+
# kick off ingest
45+
$HADOOP_HOME/bin/hadoop jar $JarFile org.sleuthkit.hadoop.pipeline.Ingest $ImageID $HdfsImage $JsonFile $FriendlyName
46+
echo "done with ingest"
47+
48+
# copy reports template
49+
$HADOOP_HOME/bin/hadoop fs -cp /texaspete/templates/reports /texaspete/data/$ImageID/
50+
51+
# kick off pipeline
52+
$HADOOP_HOME/bin/hadoop jar $JarFile org.sleuthkit.hadoop.pipeline.Pipeline $ImageID $FriendlyName
53+
54+
date

‎pipeline/pom.xml

+7
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,13 @@
8989
<profiles>
9090
<profile>
9191
<id>dependency-package</id>
92+
<!-- If you need the zip file with the dependencies of this project built
93+
along with the project, uncomment the next block of code. -->
94+
<!--
95+
<activation>
96+
<activeByDefault>true</activeByDefault>
97+
</activation>
98+
-->
9299
<build>
93100
<plugins>
94101
<plugin>

‎pushtocluster.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
for addr in "$@"
2+
do
3+
scp -r txpete/ $addr:~
4+
ssh $addr "cd ~/txpete/; ./setup.sh"
5+
done

‎whirr-ec2-framework.properties

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
#
19+
# HBase CDH install
20+
#
21+
22+
whirr.cluster-name=hadoop_framework
23+
24+
whirr.instance-templates=1 zookeeper+hadoop-namenode+hadoop-jobtracker+hbase-master,1 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
25+
26+
# replication level should not be higher than number of data nodes
27+
hbase-site.dfs.replication=1
28+
29+
#custom CDH zookeeper install and configure functions
30+
whirr.zookeeper.install-function=install_cdh_zookeeper
31+
whirr.zookeeper.configure-function=configure_cdh_zookeeper
32+
33+
#custom CDH hadoop install and configure functions
34+
whirr.hadoop.install-function=install_cdh_hadoop
35+
whirr.hadoop.configure-function=configure_cdh_hadoop
36+
37+
#custom CDH hbase install and configure functions
38+
whirr.hbase.install-function=install_cdh_hbase
39+
whirr.hbase.configure-function=configure_cdh_hbase
40+
41+
whirr.provider=aws-ec2
42+
43+
whirr.identity=<EC2_IDENTITY>
44+
whirr.credential=<EC2_CREDENTIAL>
45+
46+
# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/
47+
whirr.hardware-id=c1.xlarge
48+
49+
# Ubuntu 10.04 LTS Lucid. See http://cloud.ubuntu.com/ami/
50+
whirr.image-id=us-east-1/ami-04c9306d
51+
52+
# If you choose a different location, make sure whirr.image-id is updated too
53+
whirr.location-id=us-east-1
54+
55+
# By default use the user system SSH keys. Override them here.
56+
whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
57+
whirr.public-key-file=${sys:user.home}/.ssh/id_rsa.pub

0 commit comments

Comments
 (0)