optimality constraint for symbolic regression

bin/rational.py

@@ -17,7 +17,7 @@
 from dreamcoder.utilities import testTrainSplit, eprint, numberOfCPUs
 
 
-def makeTask(name, f):
+def makeTask(name, f, actualParameters):
     xs = [x / 100. for x in range(-500, 500)]
 
     maximum = 10
@@ -45,6 +45,7 @@ def makeTask(name, f):
                                restarts=360, steps=50,
                                likelihoodThreshold=-0.05,
                                temperature=0.1,
+                               actualParameters=actualParameters,
                                maxParameters=6,
                                loss=squaredErrorLoss)
         t.f = f
@@ -115,7 +116,8 @@ def f(x):
 def randomRational():
     no = random.choice([0, 1])
     nn, n = randomPolynomial(no)
-    dn, d = randomFactored(random.choice([1, 2]))
+    nf = random.choice([1, 2])
+    dn, d = randomFactored(nf)
 
     def f(x): return n(x) / d(x)
 
@@ -124,7 +126,7 @@ def f(x): return n(x) / d(x)
     else:
         name = "(%s)/[%s]" % (nn, dn)
 
-    return name, f
+    return name, f, no + 1 + nf
 
 
 def randomPower():
@@ -210,33 +212,33 @@ def makeTasks():
     ts = []
     while len(ts) < tasksPerType:
         n, f = randomOffset()
-        if makeTask(n, f) is None:
+        if makeTask(n, f, 1) is None:
             continue
-        ts.append(makeTask(n, f))
+        ts.append(makeTask(n, f, 1))
     tasks += ts
 
     for o in range(1, 5):
         ts = []
         while len(ts) < tasksPerType:
             n, f = randomPolynomial(o)
-            if makeTask(n, f) is None:
+            if makeTask(n, f, o + 1) is None:
                 continue
-            ts.append(makeTask(n, f))
+            ts.append(makeTask(n, f, o + 1))
         tasks += ts
     ts = []
     while len(ts) < tasksPerType * 3:
-        n, f = randomRational()
-        if makeTask(n, f) is None:
+        n, f, df = randomRational()
+        if makeTask(n, f, df) is None:
             continue
-        ts.append(makeTask(n, f))
+        ts.append(makeTask(n, f, df))
     tasks += ts
 
     ts = []
     while len(ts) < tasksPerType:
         n, f = randomPower()
-        if makeTask(n, f) is None:
+        if makeTask(n, f, 1) is None:
             continue
-        ts.append(makeTask(n, f))
+        ts.append(makeTask(n, f, 1))
     tasks += ts
     return tasks
 
@@ -276,7 +278,7 @@ def taskOfProgram(self, p, t):
         p = p.visit(RandomParameterization.single)
 
         def f(x): return p.runWithArguments([x])
-        t = makeTask(str(p), f)
+        t = makeTask(str(p), f, None)
         if t is None:
             return None
         t.features = drawFunction(200, 5., t.f)

dreamcoder/task.py

@@ -140,10 +140,11 @@ class DifferentiableTask(Task):
 
     def __init__(self, name, request, examples, _=None,
                  features=None, BIC=1., loss=None, likelihoodThreshold=None,
-                 steps=50, restarts=300, lr=0.5, decay=0.5, grow=1.2,
+                 steps=50, restarts=300, lr=0.5, decay=0.5, grow=1.2, actualParameters=None,
                  temperature=1., maxParameters=None, clipLoss=None, clipOutput=None):
         assert loss is not None
         self.temperature = temperature
+        self.actualParameters = actualParameters
         self.maxParameters = maxParameters
         self.loss = loss
         self.BIC = BIC
@@ -156,6 +157,7 @@ def __init__(self, name, request, examples, _=None,
                      "lossThreshold": -likelihoodThreshold}
         if clipLoss is not None: arguments['clipLoss'] = float(clipLoss)
         if clipOutput is not None: arguments['clipOutput'] = float(clipOutput)
+        if actualParameters is not None: arguments['actualParameters'] = int(actualParameters)
 
         self.specialTask = ("differentiable",
                             arguments)
@@ -175,6 +177,9 @@ def logLikelihood(self, e, timeout=None):
         if self.maxParameters is not None and len(
                 parameters) > self.maxParameters:
             return NEGATIVEINFINITY
+        if self.actualParameters is not None and len(
+                parameters) > self.actualParameters:
+            return NEGATIVEINFINITY
         f = e.evaluate([])
 
         loss = sum(self.loss(self.predict(f, xs), y)

official_figures

@@ -6,7 +6,7 @@ python bin/graphs.py --checkpoints `grep  -h 'it=20.*graph=True.pickle' jobs/tex
 
 ### Text figures - time and hits split
 python bin/graphs.py --x-label "" -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_hits.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*` cyan `grep -h  it=20 jobs/baseline_ec_text*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 31 NS 0
-python bin/graphs.py -i 20 --testingTimeout 600 --export figures/text_time.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*` --onlyTime
+python bin/graphs.py -i 20 --testingTimeout 600 --export figures/text_time.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*` cyan `grep -h  it=20 jobs/baseline_ec_text*|sed 's/Exported checkpoint to//g'` --title "Text editing" --onlyTime --averageColors -c 
 python bin/graphs.py -i 20 --testingTimeout 600 --title "Text editing" --export figures/text_hits_ws.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/text_ellisk_2019-01-2*` purple `grep  -h 'it=20.*.pickle' jobs/text_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/text_no_generative_ellisk_2019-02-14*`  cyan `grep -h  it=20 jobs/baseline_ec_text*|sed 's/Exported checkpoint to//g'` --noTime --baseline enum 31 NS 0
 
 ### List figures - time and hits split
@@ -15,23 +15,23 @@ python bin/graphs.py -i 20 --testingTimeout 600 --export figures/list_time.png -
 
 ### List figures - hard testing tasks only
 python bin/graphs.py -i 20 --testingTimeout 600 --title "List processing" --export figures/list_hard_hits_ws.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*`  cyan `grep -h  it=20 jobs/baseline_ec_list*|sed 's/Exported checkpoint to//g'`   --noTime --baseline enum 22 NS 15
-python bin/graphs.py -i 20 --testingTimeout 600 --export figures/list_hard_time.png --checkpoints  teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*`   --onlyTime
+python bin/graphs.py -i 20 --testingTimeout 600 --export figures/list_hard_time.png --checkpoints  teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*` cyan `grep -h  it=20 jobs/baseline_ec_list*|sed 's/Exported checkpoint to//g'` --title "List processing"  --onlyTime -c --averageColors
 python bin/graphs.py -i 20 --testingTimeout 600 --title "List processing" --export figures/list_hard_hits_ws.png --checkpoints teal `grep  -h 'it=20.*graph=True.pickle' jobs/list_hard_test_ellisk*|head  -n 5` purple `grep  -h 'it=20.*.pickle' jobs/list_hard_test_no_recognition_ellisk*|sed 's/Exported checkpoint to//g'` orange `grep  -h it=20.*graph jobs/list_hard_test_no_dsl_ellisk*`  --noTime 31 NS 15
 
 ### LOGO figures - time and hits split
 python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep  -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep  -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_logo*|sed 's/Exported checkpoint to//g'`  --export figures/logo_hits.png --noTime -i 20 --x-label "" --title "LOGO Graphics" --baseline enum 25 NS 1.2
-python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/logo_auxiliary_batch_40_1h*` orange `grep  -h it=13.*graph jobs/logo_no_dsl_auxiliary_batch_40_1h_ellisk_2019-02-11*` purple `grep  -h it=11.*pickle jobs/logo_no_recognition_batch_40_1h_ellisk*|sed 's/Exported checkpoint to//g'` --export figures/logo_time.png --onlyTime -i 11   --testingTimeout 600
+python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep  -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep  -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_logo*|sed 's/Exported checkpoint to//g'` --export figures/logo_time.png --onlyTime -i 20   --testingTimeout 600 --title "LOGO graphics" -c  --averageColors
 python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/logo_batch_50_1h_ell*` orange `grep  -h it=20.*graph jobs/logo_no_dsl_auxiliary_batch_50_1h*` purple `grep  -h it=20.*pickle jobs/logo_no_recognition_batch_50_1h*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_logo*|sed 's/Exported checkpoint to//g'` --export figures/logo_hits_ws.png --noTime -i 20 --title "LOGO Graphics" --baseline enum 25 NS 1.2
 
 ### Rational figures - time and hits split
 python bin/graphs.py --checkpoint teal `grep  -h it=20.*graph jobs/rational_batch*` orange `grep  -h it=20.*graph jobs/rational_no_dsl*` purple `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/rational_baseline_ec*|sed 's/Exported checkpoint to//g'` --noTime --x-label "" --title "Symbolic Regression" --export figures/rational_hits.png --baseline enum 65  NS 69 -i 20
-python bin/graphs.py --checkpoint teal `grep  -h it=20.*graph jobs/rational_batch*` orange `grep  -h it=20.*graph jobs/rational_no_dsl*` purple `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` --onlyTime --export figures/rational_time.png --testingTimeout 600
+python bin/graphs.py --checkpoint teal `grep  -h it=20.*graph jobs/rational_batch*` orange `grep  -h it=20.*graph jobs/rational_no_dsl*` purple `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/rational_baseline_ec*|sed 's/Exported checkpoint to//g'`  --onlyTime --export figures/rational_time.png  --testingTimeout 600 -i 20 --title "Symbolic regression" -c  --averageColors
 python bin/graphs.py --checkpoint teal `grep  -h it=20.*graph jobs/rational_batch*` orange `grep  -h it=20.*graph jobs/rational_no_dsl*` purple `grep  -h it=20.*pickle jobs/rational_noRecognition*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/rational_baseline_ec*|sed 's/Exported checkpoint to//g'` --noTime --title "Symbolic Regression" --export figures/rational_hits_ws.png --baseline enum 65  NS 69 -i 20
 
 ### Tower figures - time and hits split
 # actual baseline results are both 9, separated for visibility
 python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep  -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_tower*|sed 's/Exported checkpoint to//g'` --export figures/tower_hits.png --noTime --x-label "" --title "Tower building" --baseline enum 8 NS 10 -i 20
-python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep  -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` --export figures/tower_time.png --onlyTime --testingTimeout 600 -i 20
+python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep  -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_tower*|sed 's/Exported checkpoint to//g'` --export figures/tower_time.png --onlyTime --testingTimeout 600 -i 20  --title "Tower building" --averageColors -c 
 python bin/graphs.py --checkpoints teal `grep  -h it=20.*graph jobs/tower_batch_50_3600*` orange `grep  -h it=20.*graph jobs/tower_no_dsl_batch_50_3600*` purple `grep  -h it=20.*pickle jobs/tower_no_recognition_batch_50_3600*|sed 's/Exported checkpoint to//g'` cyan `grep -h  it=20 jobs/baseline_ec_tower*|sed 's/Exported checkpoint to//g'` --export figures/tower_hits_ws.png --noTime --title "Tower building" --baseline enum 8 NS 10  -i 20
 
 ### Regex figures - time and hits split

solvers/task.ml

@@ -207,6 +207,7 @@ register_special_task "differentiable"
     let temperature = maybe_float "temperature" 1. in
     let parameterPenalty = maybe_float "parameterPenalty" 0. in
     let maxParameters = maybe_int "maxParameters" 99 in
+    let actualParameters = maybe_int "maxParameters" 99 in
     let restarts = maybe_int "restarts" 300 in
     let steps = maybe_int "steps" 50 in
     let lr = maybe_float "lr" 0.5 in
@@ -235,7 +236,7 @@ register_special_task "differentiable"
       (fun expression ->
          let (p,parameters) = replace_placeholders expression in
          assert (List.length parameters <= maxParameters);
-        if List.length parameters > maxParameters then log 0. else 
+        if List.length parameters > maxParameters || List.length parameters > actualParameters then log 0. else 
           let p = analyze_lazy_evaluation p in
           (* let predictions = examples |> List.map ~f:(fun (xs,_) -> *)
           (*     run_for_interval timeout (fun () -> run_lazy_analyzed_with_arguments p xs)) *)