final changes to code

minrule-extraction/combine_minimal_grammars.py

@@ -13,7 +13,7 @@
 Arg2: directory where guaranteed binary minimal rules are
 Arg3: directory where ZGC rules are
 Arg4: output directory
-Usage: 
+Usage: python combine_minimal_grammars.py invalid_rules guaranteed_binary_dir/  ZGC_rules_dir/   output_dir/
 '''
 
 import sys, commands, string, os, gzip, re

minrule-extraction/featurize_rules.py

@@ -198,7 +198,6 @@ def decorateSentenceGrammar(minRule_file, hiero_file, out_file, lex_model, optDi
                 out_fh.write("%s\n"%ruleToPrint)
         else:
             seen_rules.extend(rules_output)  #global write
-            #print "for grammar %s, out of %d rules, %d are also in hiero"%(minRule_file, numRulesTotal, numRulesInHiero)
         print "Grammar %s featurization complete: %d rules"%(minRule_file, numRulesTotal)
     if perSent and not marginal: #add the NT only rules, but only if we're not reading in marginals (since we already have the NT rules), then close
         out_fh.write("[X] ||| [X,1] [X,2] ||| [1] [2] ||| Glue=1\n")
@@ -242,8 +241,8 @@ def main():
     else:
         pool = mp.Pool(numProcesses)
     for minRule_file in minRule_grammars:
-        #decorateSentenceGrammar(minRule_grammars_loc + minRule_file, hiero_grammars_loc + minRule_file, outFile_loc + minRule_file, lex_model, optDict)
-        pool.apply_async(decorateSentenceGrammar, (minRule_grammars_loc + minRule_file, hiero_grammars_loc + minRule_file, outFile_loc + minRule_file, lex_model, optDict))    
+        decorateSentenceGrammar(minRule_grammars_loc + minRule_file, hiero_grammars_loc + minRule_file, outFile_loc + minRule_file, lex_model, optDict)
+        #pool.apply_async(decorateSentenceGrammar, (minRule_grammars_loc + minRule_file, hiero_grammars_loc + minRule_file, outFile_loc + minRule_file, lex_model, optDict))    
     pool.close()
     pool.join()                             
 

parameter-estimation/count_estimation.py

@@ -1,9 +1,13 @@
 #!/usr/bin/python -tt
 
 '''
-Description for file here. 
-
-28 May: Removed addone functionality, shifted it to featurize_rules.py
+File: count_estimation.py
+Date: May 25, 2014
+Description: simple file that, given a 'full' directory (namely, 
+a derivation tree for every sentence pair with the rules for every
+NT written out), estimates the counts for every rule.  In this   code, 
+we filter all rules with number of NTs > 2 on the RHS.  
+Update: 28 May: Removed addone functionality, shifted it to featurize_rules.py
 '''
 
 import sys, commands, string, cPickle, os, gzip, re, getopt
@@ -72,13 +76,28 @@ def IncrementCountsByLine(filehandle):
 
 def WriteOutCounts(fileout):
     reformatted = {}
+    binary = 0
+    unary = 0
+    preterm = 0
     for rule in Counts:
+        arity = CheckArity(rule) #first, keep track of the counts of rules across pre-term, unary, and binary
+        if arity == 0: 
+            preterm += 1
+        elif arity == 1:
+            unary += 1
+        elif arity == 2:
+            binary += 1
+        else:
+            sys.stderr.write("Error! Writing out rule with # of NTs on RHS > 2!\n")
         srcKey = ' ||| '.join(rule.split(' ||| ')[:-1])
         tgtKey = rule.split(' ||| ')[-1]
         srcDict = reformatted[srcKey] if srcKey in reformatted else {}
         srcDict[tgtKey] = Counts[rule]
         reformatted[srcKey] = srcDict
     cPickle.dump(reformatted, open(fileout, "wb"))
+    print "Number of pre-terminal rules (including singletons): %d"%preterm
+    print "Number of unary rules: %d"%unary
+    print "Number of binary rules: %d"%binary
 
 def main():
     (opts, args) = getopt.getopt(sys.argv[1:], 'n:')