update dreamcoder

dreamcoder/domains/list/main.py

@@ -260,24 +260,23 @@ def list_options(parser):
     parser.add_argument("--random-seed", type=int, default=17)
 
 
-def main(args):
+def main(dataset='Lucas-old', maxTasks=10_000):
     """
     Takes the return value of the `commandlineArguments()` function as input and
     trains/tests the model on manipulating sequences of numbers.
     """
-    random.seed(args.pop("random_seed"))
+    random.seed(9)
 
-    dataset = args.pop("dataset")
     tasks = {
         "Lucas-old": lambda: retrieveJSONTasks("data/list_tasks.json") + sortBootstrap(),
         "bootstrap": make_list_bootstrap_tasks,
         "sorting": sortBootstrap,
-        "Lucas-depth1": lambda: retrieveJSONTasks("data/list_tasks2.json")[:105],
-        "Lucas-depth2": lambda: retrieveJSONTasks("data/list_tasks2.json")[:4928],
-        "Lucas-depth3": lambda: retrieveJSONTasks("data/list_tasks2.json"),
+        # removed as file over 10MB
+        # "Lucas-depth1": lambda: retrieveJSONTasks("data/list_tasks2.json")[:105],
+        # "Lucas-depth2": lambda: retrieveJSONTasks("data/list_tasks2.json")[:4928],
+        # "Lucas-depth3": lambda: retrieveJSONTasks("data/list_tasks2.json"),
     }[dataset]()
 
-    maxTasks = args.pop("maxTasks")
     if maxTasks and len(tasks) > maxTasks:
         necessaryTasks = []  # maxTasks will not consider these
         if dataset.startswith("Lucas2.0") and dataset != "Lucas2.0-depth1":
@@ -344,67 +343,4 @@ def main(args):
         return all( len(xs) == 1 and xs[0] == y for xs, y in t.examples  )
     eprint("Removed", sum(isIdentityTask(t) for t in tasks), "tasks that were just the identity function")
     tasks = [t for t in tasks if not isIdentityTask(t) ]
-
-    prims = {"base": basePrimitives,
-             "McCarthy": McCarthyPrimitives,
-             "common": bootstrapTarget_extra,
-             "noLength": no_length,
-             "rich": primitives}[args.pop("primitives")]()
-    haveLength = not args.pop("noLength")
-    haveMap = not args.pop("noMap")
-    haveUnfold = not args.pop("noUnfold")
-    eprint(f"Including map as a primitive? {haveMap}")
-    eprint(f"Including length as a primitive? {haveLength}")
-    eprint(f"Including unfold as a primitive? {haveUnfold}")
-    baseGrammar = Grammar.uniform([p
-                                   for p in prims
-                                   if (p.name != "map" or haveMap) and \
-                                   (p.name != "unfold" or haveUnfold) and \
-                                   (p.name != "length" or haveLength)])
-
-    extractor = {
-        "learned": LearnedFeatureExtractor,
-    }[args.pop("extractor")]
-    extractor.H = args.pop("hidden")
-
-    timestamp = datetime.datetime.now().isoformat()
-    outputDirectory = "experimentOutputs/list/%s"%timestamp
-    os.system("mkdir -p %s"%outputDirectory)
-    
-    args.update({
-        "featureExtractor": extractor,
-        "outputPrefix": "%s/list"%outputDirectory,
-        "evaluationTimeout": 0.0005,
-    })
-    
-
-    eprint("Got {} list tasks".format(len(tasks)))
-    split = args.pop("split")
-    if split:
-        train_some = defaultdict(list)
-        for t in tasks:
-            necessary = train_necessary(t)
-            if not necessary:
-                continue
-            if necessary == "some":
-                train_some[t.name.split()[0]].append(t)
-            else:
-                t.mustTrain = True
-        for k in sorted(train_some):
-            ts = train_some[k]
-            random.shuffle(ts)
-            ts.pop().mustTrain = True
-
-        test, train = testTrainSplit(tasks, split)
-        if True:
-            test = [t for t in test
-                    if t.name not in EASYLISTTASKS]
-
-        eprint(
-            "Alotted {} tasks for training and {} for testing".format(
-                len(train), len(test)))
-    else:
-        train = tasks
-        test = []
-
-    explorationCompression(baseGrammar, train, testingTasks=test, **args)
+    return tasks

dreamcoder/domains/list/makeListTasks.py

@@ -85,7 +85,7 @@ def make_list_task(name, examples, **params):
     yield Task(name, program_type, examples, cache=cache)
 
 
-def make_list_tasks(n_examples):
+def make_list_tasks(n_examples=4):
     import listroutines as lr
 
     for routine in lr.find(count=100):  # all routines

dreamcoder/domains/text/makeTextTasks.py

@@ -91,12 +91,10 @@ def randomWords(ds, minimum=1, lb=2, ub=4):
     return s
 
 
-def makeTasks():
+def makeTasks(n_examples = 4):
     import random
     random.seed(9)
 
-    NUMBEROFEXAMPLES = 4
-
     problems = []
 
     def toList(s): return [c for c in s]
@@ -108,7 +106,7 @@ def makeTasks():
         if isinstance(x, list):
             return [preprocess(z) for z in x]
         if isinstance(x, str):
-            return [c for c in x]
+            return x
         if isinstance(x, bool):
             return x
         assert False
@@ -126,25 +124,25 @@ def makeTasks():
         if d1 != d2:
             problem("Replace '%s' w/ '%s'" % (d1, d2),
                     [((x,), x.replace(d1, d2))
-                     for _ in range(NUMBEROFEXAMPLES)
+                     for _ in range(n_examples)
                      for x in [randomWords(d1)]],
                     needToTrain=False)
     for d in delimiters:
         problem("drop first word delimited by '%s'" % d,
                 [((x,), d.join(x.split(d)[1:]))
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for x in [randomWords(d)]],
                 needToTrain=True)
         for n in [0, 1, -1]:
             problem("nth (n=%d) word delimited by '%s'" % (n, d),
                     [((x,), x.split(d)[n])
-                     for _ in range(NUMBEROFEXAMPLES)
+                     for _ in range(n_examples)
                      for x in [randomWords(d)]],
                     needToTrain=True)
     for d1 in delimiters:
         problem("Append two words delimited by '%s'" % (d1),
                 [((x, y), x + d1 + y)
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for x in [randomWord()]
                  for y in [randomWord()]],
                 needToTrain=True)
@@ -154,20 +152,20 @@ def makeTasks():
             :len(delimiters)]:
         problem("Append two words delimited by '%s%s'" % (d1, d2),
                 [((x, y), x + d1 + d2 + y)
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for x in [randomWord()]
                  for y in [randomWord()]],
                 needToTrain=True)
     for n in range(1, 6):
         problem("Drop last %d characters" % n,
                 [((x,), x[:-n])
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for x in [randomWord(minimum=n)]],
                 needToTrain=True)
         if n > 1:
             problem("Take first %d characters" % n,
                     [((x,), x[:n])
-                     for _ in range(NUMBEROFEXAMPLES)
+                     for _ in range(n_examples)
                      for x in [randomWord(minimum=n)]],
                     needToTrain=True)
     for d1, d2 in randomPermutation(
@@ -176,7 +174,7 @@ def makeTasks():
             :len(delimiters)]:
         problem("Extract word delimited by '%s' - '%s'" % (d1, d2),
                 [((a + d1 + b + d2 + c + d + e,), b)
-                 for _ in range(int(NUMBEROFEXAMPLES / 2))
+                 for _ in range(int(n_examples / 2))
                  for d in [d1, d2]
                  for a in [randomWord()]
                  for b in [randomWord()]
@@ -187,7 +185,7 @@ def makeTasks():
     for n in range(len(delimiters)):
         problem("First letters of words (%s)" % ("I" * (1 + n)),
                 [((x,), "".join(map(lambda z: z[0], x.split(' '))))
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for x in [randomWords(' ')]
                  ],
                 needToTrain=True)
@@ -195,27 +193,27 @@ def makeTasks():
     for d in delimiters:
         problem("Take first character and append '%s'" % d,
                 [((x,), x[0] + d)
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for x in [randomWord()]],
                 needToTrain=True)
 
     for n in range(len(delimiters)):
         problem("Abbreviate separate words (%s)" % ("I" * (n + 1)),
                 [((x, y), "%s.%s." % (x[0], y[0]))
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for y in [randomWord()]
                  for x in [randomWord()]])
         d = delimiters[n]
         problem("Abbreviate words separated by '%s'" % d,
                 [((x + d + y,), "%s.%s." % (x[0], y[0]))
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for y in [randomWord()]
                  for x in [randomWord()]])
 
     for n in range(len(delimiters)):
         problem("Append 2 strings (%s)" % ('I' * (n + 1)),
                 [((x, y), x + y)
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for y in [randomWord()]
                  for x in [randomWord()]],
                 needToTrain=True)
@@ -224,33 +222,33 @@ def makeTasks():
         w = randomWord(minimum=3)
         problem("Prepend '%s'" % w,
                 [((x,), w + x)
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for x in [randomWord()]])
         w = randomWord(minimum=3)
         problem("Append '%s'" % w,
                 [((x,), x + w)
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for x in [randomWord()]])
         w = randomWord(minimum=3)
         problem("Prepend '%s' to first word" % w,
                 [((x + ' ' + y,), w + x)
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for x in [randomWord()]
                  for y in [randomWord()]])
 
     for n in range(1,6):
         problem("parentheses around a single word (%s)"%('I'*n),
                 [((w,),"(%s)"%w)
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for w in [randomWord()] ])
     problem("parentheses around first word",
             [((w + " " + s,),"(%s)"%w)
-             for _ in range(NUMBEROFEXAMPLES)
+             for _ in range(n_examples)
              for w in [randomWord()]
              for s in [randomWords(" ")] ])
     problem("parentheses around second word",
             [((s,), "(%s)"%(s.split(" ")[1]))
-             for _ in range(NUMBEROFEXAMPLES)
+             for _ in range(n_examples)
              for s in [randomWords(" ")] ])
 
     allowed = [d for d in delimiters if d not in "()"]
@@ -258,7 +256,7 @@ def makeTasks():
         problem("parentheses around word delimited by '%s' & '%s'"%(d1,d2),
                 [((prefix + d1 + word + d2 + suffix,),
                   prefix + d1 + '(' + word + ')' + d2 + suffix)
-                 for _ in range(NUMBEROFEXAMPLES)
+                 for _ in range(n_examples)
                  for prefix in [randomWords("", lb=0, ub=1)]
                  for suffix in [randomWords(allowed, ub=2, lb=1)]
                  for word in [randomWord()] ])
@@ -267,7 +265,7 @@ def makeTasks():
         w = randomWord(minimum=3)
         problem("ensure suffix `%s`"%w,
                 [ ((s + (w if f else ""),), s + w)
-                  for _ in range(NUMBEROFEXAMPLES)
+                  for _ in range(n_examples)
                   for s in [randomWords(" ")]
                   for f in [random.choice([True,False])] ])