Skripts for lmi testing

create_screens.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+COUNTER=0
+MAXIMUM=20
+LINES=12741
+
+while [ $COUNTER -le $MAXIMUM ]; do
+
+    let COUNTER2=$COUNTER+1
+
+    screen -dmS "teamlab"$COUNTER2 ./run1.sh $(($COUNTER*$LINES)) $(($COUNTER2*$LINES))
+
+    let COUNTER=COUNTER+1
+
+done
+
+
+

kill_screens.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+COUNTER=1
+MAXIMUM=100
+
+while [ $COUNTER -le $MAXIMUM ]; do
+
+    screen -S "teamlab"$COUNTER -X quit
+
+    let COUNTER=COUNTER+1
+
+done
+
+
+
+
+

run.sh

@@ -8,40 +8,38 @@ EVALUATIONS="evaluations/"
 
 affixes=0
 train=1
-test=1
-evaluate=1
+test=0
+evaluate=0
 
-head -20000 $CORPORA/train.col >> $CORPORA/train_top5000.col
-head -20000 $CORPORA/dev.col >> $CORPORA/dev_top5000.col
+#head -20000 $CORPORA/train.col >> $CORPORA/train_top5000.col
+#head -20000 $CORPORA/dev.col >> $CORPORA/dev_top5000.col
+#1178
 
 # Finding possible features, e.g. Affixes.
 if [ "$affixes" = 1 ]; then
     python -u tagger.py -feat -i $CORPORA/train.col
 fi
-COUNTER=0
-while read p; do
-#  Train the model
-    let COUNTER=COUNTER+1
-    if [ "$train" = 1 ]; then
-        #python -u tagger.py -train -i $CORPORA/train.col -t $p -e 5 -m $MODELS/model$COUNTER
-        python -u tagger.py -train -i $CORPORA/train_top5000.col -t $p -e 5 -m $MODELS/model$COUNTER
-        #python tagger.py -train -i $CORPORA/train_top5000.col -e 5 -m model
-    fi
-
-    # Test the model
-    if [ "$test" = 1 ]; then
-        #python -u tagger.py -test -i $CORPORA/dev.col -m $MODELS/model$COUNTER -o $PREDICTIONS/prediction$COUNTER.col
-        python -u tagger.py -test -i $CORPORA/dev_top5000.col -m $MODELS/model$COUNTER -o $PREDICTIONS/prediction$COUNTER.col
-        #python tagger.py -test -i $CORPORA/dev_top5000.col -m model -o prediction.col
-        rm -f $MODELS/model$COUNTER
-    fi
-
-    # Evaluate the results
-    if [ "$evaluate" = 1 ]; then
-        python -u tagger.py -ev -i $PREDICTIONS/prediction$COUNTER.col -o $EVALUATIONS/evaluation$COUNTER.txt
-        #python -u tagger.py -ev -i $CORPORA/test_stuff/nn.col -o evaluation.txt
-        #python -u tagger.py -ev -i $CORPORA/test_stuff/leer.col -o evaluation.txt
-    fi
-done <lmi.txt
-rm $CORPORA/train_top5000.col
-rm $CORPORA/dev_top5000.col
+
+if [ "$train" = 1 ]; then
+    python -u tagger.py -train -i $CORPORA/train.col -t 0 -e 5 -m model
+    #python -u tagger.py -train -i $CORPORA/train_top5000.col -t $p -e 5 -m $MODELS/model$COUNTER
+    #python tagger.py -train -i $CORPORA/train_top5000.col -e 5 -m model
+fi
+
+# Test the model
+if [ "$test" = 1 ]; then
+    python -u tagger.py -test -i $CORPORA/dev.col -m model -o prediction.col
+    #python -u tagger.py -test -i $CORPORA/dev_top5000.col -m $MODELS/model$COUNTER -o $PREDICTIONS/prediction$COUNTER.col
+    #python tagger.py -test -i $CORPORA/dev_top5000.col -m model -o prediction.col
+    rm -f $MODELS/model$COUNTER
+fi
+
+# Evaluate the results
+if [ "$evaluate" = 1 ]; then
+    python -u tagger.py -ev -i prediction.col -o evaluation.txt
+    #python -u tagger.py -ev -i $CORPORA/test_stuff/nn.col -o evaluation.txt
+    #python -u tagger.py -ev -i $CORPORA/test_stuff/leer.col -o evaluation.txt
+fi
+
+#rm $CORPORA/train_top5000.col
+#rm $CORPORA/dev_top5000.col

run1.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+CORPORA="../team-lab-ss2015/data/pos"
+
+MODELS="models"
+PREDICTIONS="predictions"
+EVALUATIONS="/mount/studenten/projekt-cl/WS-2013-2014/student-workspace/nedderyk/teamlabproject15/evaluations"
+
+affixes=0
+train=1
+test=1
+evaluate=1
+
+#head -20000 $CORPORA/train.col >> $CORPORA/train_top5000.col
+#head -20000 $CORPORA/dev.col >> $CORPORA/dev_top5000.col
+
+# Finding possible features, e.g. Affixes.
+if [ "$affixes" = 1 ]; then
+    python -u tagger.py -feat -i $CORPORA/train.col
+fi
+
+COUNTER=0
+
+while read p; do
+#  Train the model
+    let COUNTER=COUNTER+1
+
+    if [ $COUNTER -ge $1 ]  && [ $COUNTER -lt $2 ]; then
+
+
+        if [ "$train" = 1 ]; then
+            python -u tagger.py -train -i $CORPORA/train.col -t $p -e 5 -m $MODELS/model$COUNTER
+            #python -u tagger.py -train -i $CORPORA/train_top5000.col -t $p -e 5 -m $MODELS/model$COUNTER
+            #python tagger.py -train -i $CORPORA/train_top5000.col -e 5 -m model
+        fi
+
+        # Test the model
+        if [ "$test" = 1 ]; then
+            python -u tagger.py -test -i $CORPORA/dev.col -m $MODELS/model$COUNTER -o $PREDICTIONS/prediction$COUNTER.col
+            #python -u tagger.py -test -i $CORPORA/dev_top5000.col -m $MODELS/model$COUNTER -o $PREDICTIONS/prediction$COUNTER.col
+            #python tagger.py -test -i $CORPORA/dev_top5000.col -m model -o prediction.col
+            rm -f $MODELS/model$COUNTER
+        fi
+
+        # Evaluate the results
+        if [ "$evaluate" = 1 ]; then
+            python -u tagger.py -ev -i $PREDICTIONS/prediction$COUNTER.col -o $EVALUATIONS/evaluation$COUNTER.txt
+            #python -u tagger.py -ev -i $CORPORA/test_stuff/nn.col -o evaluation.txt
+            #python -u tagger.py -ev -i $CORPORA/test_stuff/leer.col -o evaluation.txt
+            rm -f $PREDICTIONS/prediction$COUNTER.col
+        fi
+    fi
+
+done <lmi.txt
+
+#rm $CORPORA/train_top5000.col
+#rm $CORPORA/dev_top5000.col

tagger.py

@@ -82,7 +82,7 @@ def train(self, file_in, file_out, max_iterations, threshold):
         classifiers = {}
 
         lmi_calc = lmi.lmi(tokens, feat_vec)
-        lmi_dict = lmi_calc.compute_lmi
+        lmi_dict = lmi_calc.compute_lmi()
 
         # instantiate a classifier for each pos tag type:
         for tag in tag_set:

testscript.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+touch "file"$1
+touch "file"$2