Merge pull request #24 from DhananjayAshok/recover

Refactor Python into more functions, and Julia into more files.

.gitignore

@@ -8,3 +8,7 @@ trials*
 **/__pycache__
 build
 dist
+*.vs/*
+*.pyproj
+*.sln
+pysr/.vs/

README.md

@@ -69,9 +69,10 @@ pip install pysr
 
 # Quickstart
 
+Here is some demo code (also found in `example.py`)
 ```python
 import numpy as np
-from pysr import pysr, best, get_hof
+from pysr import pysr, best
 
 # Dataset
 X = 2*np.random.randn(100, 5)
@@ -108,4 +109,3 @@ This is a pandas table, with additional columns:
 - `score` - a metric akin to Occam's razor; you should use this to help select the "true" equation.
 - `sympy_format` - sympy equation.
 - `lambda_format` - a lambda function for that equation, that you can pass values through.
-

example.py

@@ -0,0 +1,17 @@
+import numpy as np
+from pysr import pysr, best
+
+# Dataset
+X = 2*np.random.randn(100, 5)
+y = 2*np.cos(X[:, 3]) + X[:, 0]**2 - 2
+
+# Learn equations
+equations = pysr(X, y, niterations=5,
+    binary_operators=["plus", "mult"],
+    unary_operators=[
+      "cos", "exp", "sin", #Pre-defined library of operators (see https://pysr.readthedocs.io/en/latest/docs/operators/)
+      "inv(x) = 1/x"]) # Define your own operator! (Julia syntax)
+
+...# (you can use ctl-c to exit early)
+
+print(best(equations))

julia/CheckConstraints.jl

@@ -0,0 +1,42 @@
+# Check if any binary operator are overly complex
+function flagBinOperatorComplexity(tree::Node, op::Int)::Bool
+    if tree.degree == 0
+        return false
+    elseif tree.degree == 1
+        return flagBinOperatorComplexity(tree.l, op)
+    else
+        if tree.op == op
+            overly_complex = (
+                    ((bin_constraints[op][1] > -1) &&
+                     (countNodes(tree.l) > bin_constraints[op][1]))
+                      ||
+                    ((bin_constraints[op][2] > -1) &&
+                     (countNodes(tree.r) > bin_constraints[op][2]))
+                )
+            if overly_complex
+                return true
+            end
+        end
+        return (flagBinOperatorComplexity(tree.l, op) || flagBinOperatorComplexity(tree.r, op))
+    end
+end
+
+# Check if any unary operators are overly complex
+function flagUnaOperatorComplexity(tree::Node, op::Int)::Bool
+    if tree.degree == 0
+        return false
+    elseif tree.degree == 1
+        if tree.op == op
+            overly_complex = (
+                      (una_constraints[op] > -1) &&
+                      (countNodes(tree.l) > una_constraints[op])
+                )
+            if overly_complex
+                return true
+            end
+        end
+        return flagUnaOperatorComplexity(tree.l, op)
+    else
+        return (flagUnaOperatorComplexity(tree.l, op) || flagUnaOperatorComplexity(tree.r, op))
+    end
+end

julia/ConstantOptimization.jl

@@ -0,0 +1,49 @@
+import Optim
+
+# Proxy function for optimization
+function optFunc(x::Array{Float32, 1}, tree::Node)::Float32
+    setConstants(tree, x)
+    return scoreFunc(tree)
+end
+
+# Use Nelder-Mead to optimize the constants in an equation
+function optimizeConstants(member::PopMember)::PopMember
+    nconst = countConstants(member.tree)
+    if nconst == 0
+        return member
+    end
+    x0 = getConstants(member.tree)
+    f(x::Array{Float32,1})::Float32 = optFunc(x, member.tree)
+    if size(x0)[1] == 1
+        algorithm = Optim.Newton
+    else
+        algorithm = Optim.NelderMead
+    end
+
+    try
+        result = Optim.optimize(f, x0, algorithm(), Optim.Options(iterations=100))
+        # Try other initial conditions:
+        for i=1:nrestarts
+            tmpresult = Optim.optimize(f, x0 .* (1f0 .+ 5f-1*randn(Float32, size(x0)[1])), algorithm(), Optim.Options(iterations=100))
+            if tmpresult.minimum < result.minimum
+                result = tmpresult
+            end
+        end
+
+        if Optim.converged(result)
+            setConstants(member.tree, result.minimizer)
+            member.score = convert(Float32, result.minimum)
+            member.birth = getTime()
+        else
+            setConstants(member.tree, x0)
+        end
+    catch error
+        # Fine if optimization encountered domain error, just return x0
+        if isa(error, AssertionError)
+            setConstants(member.tree, x0)
+        else
+            throw(error)
+        end
+    end
+    return member
+end

julia/Equation.jl

@@ -0,0 +1,173 @@
+# Define a serialization format for the symbolic equations:
+mutable struct Node
+    #Holds operators, variables, constants in a tree
+    degree::Integer #0 for constant/variable, 1 for cos/sin, 2 for +/* etc.
+    val::Union{Float32, Integer} #Either const value, or enumerates variable
+    constant::Bool #false if variable
+    op::Integer #enumerates operator (separately for degree=1,2)
+    l::Union{Node, Nothing}
+    r::Union{Node, Nothing}
+
+    Node(val::Float32) = new(0, val, true, 1, nothing, nothing)
+    Node(val::Integer) = new(0, val, false, 1, nothing, nothing)
+    Node(op::Integer, l::Node) = new(1, 0.0f0, false, op, l, nothing)
+    Node(op::Integer, l::Union{Float32, Integer}) = new(1, 0.0f0, false, op, Node(l), nothing)
+    Node(op::Integer, l::Node, r::Node) = new(2, 0.0f0, false, op, l, r)
+
+    #Allow to pass the leaf value without additional node call:
+    Node(op::Integer, l::Union{Float32, Integer}, r::Node) = new(2, 0.0f0, false, op, Node(l), r)
+    Node(op::Integer, l::Node, r::Union{Float32, Integer}) = new(2, 0.0f0, false, op, l, Node(r))
+    Node(op::Integer, l::Union{Float32, Integer}, r::Union{Float32, Integer}) = new(2, 0.0f0, false, op, Node(l), Node(r))
+end
+
+# Copy an equation (faster than deepcopy)
+function copyNode(tree::Node)::Node
+   if tree.degree == 0
+       return Node(tree.val)
+   elseif tree.degree == 1
+       return Node(tree.op, copyNode(tree.l))
+    else
+        return Node(tree.op, copyNode(tree.l), copyNode(tree.r))
+   end
+end
+
+# Count the operators, constants, variables in an equation
+function countNodes(tree::Node)::Integer
+    if tree.degree == 0
+        return 1
+    elseif tree.degree == 1
+        return 1 + countNodes(tree.l)
+    else
+        return 1 + countNodes(tree.l) + countNodes(tree.r)
+    end
+end
+
+# Count the max depth of a tree
+function countDepth(tree::Node)::Integer
+    if tree.degree == 0
+        return 1
+    elseif tree.degree == 1
+        return 1 + countDepth(tree.l)
+    else
+        return 1 + max(countDepth(tree.l), countDepth(tree.r))
+    end
+end
+
+# Convert an equation to a string
+function stringTree(tree::Node)::String
+    if tree.degree == 0
+        if tree.constant
+            return string(tree.val)
+        else
+            if useVarMap
+                return varMap[tree.val]
+            else
+                return "x$(tree.val - 1)"
+            end
+        end
+    elseif tree.degree == 1
+        return "$(unaops[tree.op])($(stringTree(tree.l)))"
+    else
+        return "$(binops[tree.op])($(stringTree(tree.l)), $(stringTree(tree.r)))"
+    end
+end
+
+# Print an equation
+function printTree(tree::Node)
+    println(stringTree(tree))
+end
+
+# Return a random node from the tree
+function randomNode(tree::Node)::Node
+    if tree.degree == 0
+        return tree
+    end
+    a = countNodes(tree)
+    b = 0
+    c = 0
+    if tree.degree >= 1
+        b = countNodes(tree.l)
+    end
+    if tree.degree == 2
+        c = countNodes(tree.r)
+    end
+
+    i = rand(1:1+b+c)
+    if i <= b
+        return randomNode(tree.l)
+    elseif i == b + 1
+        return tree
+    end
+
+    return randomNode(tree.r)
+end
+
+# Count the number of unary operators in the equation
+function countUnaryOperators(tree::Node)::Integer
+    if tree.degree == 0
+        return 0
+    elseif tree.degree == 1
+        return 1 + countUnaryOperators(tree.l)
+    else
+        return 0 + countUnaryOperators(tree.l) + countUnaryOperators(tree.r)
+    end
+end
+
+# Count the number of binary operators in the equation
+function countBinaryOperators(tree::Node)::Integer
+    if tree.degree == 0
+        return 0
+    elseif tree.degree == 1
+        return 0 + countBinaryOperators(tree.l)
+    else
+        return 1 + countBinaryOperators(tree.l) + countBinaryOperators(tree.r)
+    end
+end
+
+# Count the number of operators in the equation
+function countOperators(tree::Node)::Integer
+    return countUnaryOperators(tree) + countBinaryOperators(tree)
+end
+
+
+# Count the number of constants in an equation
+function countConstants(tree::Node)::Integer
+    if tree.degree == 0
+        return convert(Integer, tree.constant)
+    elseif tree.degree == 1
+        return 0 + countConstants(tree.l)
+    else
+        return 0 + countConstants(tree.l) + countConstants(tree.r)
+    end
+end
+
+# Get all the constants from a tree
+function getConstants(tree::Node)::Array{Float32, 1}
+    if tree.degree == 0
+        if tree.constant
+            return [tree.val]
+        else
+            return Float32[]
+        end
+    elseif tree.degree == 1
+        return getConstants(tree.l)
+    else
+        both = [getConstants(tree.l), getConstants(tree.r)]
+        return [constant for subtree in both for constant in subtree]
+    end
+end
+
+# Set all the constants inside a tree
+function setConstants(tree::Node, constants::Array{Float32, 1})
+    if tree.degree == 0
+        if tree.constant
+            tree.val = constants[1]
+        end
+    elseif tree.degree == 1
+        setConstants(tree.l, constants)
+    else
+        numberLeft = countConstants(tree.l)
+        setConstants(tree.l, constants)
+        setConstants(tree.r, constants[numberLeft+1:end])
+    end
+end

julia/EvaluateEquation.jl

@@ -0,0 +1,47 @@
+# Evaluate an equation over an array of datapoints
+function evalTreeArray(tree::Node)::Union{Array{Float32, 1}, Nothing}
+    return evalTreeArray(tree, X)
+end
+
+
+# Evaluate an equation over an array of datapoints
+function evalTreeArray(tree::Node, cX::Array{Float32, 2})::Union{Array{Float32, 1}, Nothing}
+    clen = size(cX)[1]
+    if tree.degree == 0
+        if tree.constant
+            return fill(tree.val, clen)
+        else
+            return copy(cX[:, tree.val])
+        end
+    elseif tree.degree == 1
+        cumulator = evalTreeArray(tree.l, cX)
+        if cumulator === nothing
+            return nothing
+        end
+        op_idx = tree.op
+        UNAOP!(cumulator, op_idx, clen)
+        @inbounds for i=1:clen
+            if isinf(cumulator[i]) || isnan(cumulator[i])
+                return nothing
+            end
+        end
+        return cumulator
+    else
+        cumulator = evalTreeArray(tree.l, cX)
+        if cumulator === nothing
+            return nothing
+        end
+        array2 = evalTreeArray(tree.r, cX)
+        if array2 === nothing
+            return nothing
+        end
+        op_idx = tree.op
+        BINOP!(cumulator, array2, op_idx, clen)
+        @inbounds for i=1:clen
+            if isinf(cumulator[i]) || isnan(cumulator[i])
+                return nothing
+            end
+        end
+        return cumulator
+    end
+end

julia/LossFunctions.jl

@@ -0,0 +1,82 @@
+import Random: randperm
+
+# Sum of square error between two arrays
+function SSE(x::Array{Float32}, y::Array{Float32})::Float32
+    diff = (x - y)
+    return sum(diff .* diff)
+end
+function SSE(x::Nothing, y::Array{Float32})::Float32
+    return 1f9
+end
+
+# Sum of square error between two arrays, with weights
+function SSE(x::Array{Float32}, y::Array{Float32}, w::Array{Float32})::Float32
+    diff = (x - y)
+    return sum(diff .* diff .* w)
+end
+function SSE(x::Nothing, y::Array{Float32}, w::Array{Float32})::Float32
+    return Nothing
+end
+
+# Mean of square error between two arrays
+function MSE(x::Nothing, y::Array{Float32})::Float32
+    return 1f9
+end
+
+# Mean of square error between two arrays
+function MSE(x::Array{Float32}, y::Array{Float32})::Float32
+    return SSE(x, y)/size(x)[1]
+end
+
+# Mean of square error between two arrays
+function MSE(x::Nothing, y::Array{Float32}, w::Array{Float32})::Float32
+    return 1f9
+end
+
+# Mean of square error between two arrays
+function MSE(x::Array{Float32}, y::Array{Float32}, w::Array{Float32})::Float32
+    return SSE(x, y, w)/sum(w)
+end
+
+if weighted
+    const avgy = sum(y .* weights)/sum(weights)
+    const baselineMSE = MSE(y, convert(Array{Float32, 1}, ones(len) .* avgy), weights)
+else
+    const avgy = sum(y)/len
+    const baselineMSE = MSE(y, convert(Array{Float32, 1}, ones(len) .* avgy))
+end
+
+# Score an equation
+function scoreFunc(tree::Node)::Float32
+    prediction = evalTreeArray(tree)
+    if prediction === nothing
+        return 1f9
+    end
+    if weighted
+        mse = MSE(prediction, y, weights)
+    else
+        mse = MSE(prediction, y)
+    end
+    return mse / baselineMSE + countNodes(tree)*parsimony
+end
+
+# Score an equation with a small batch
+function scoreFuncBatch(tree::Node)::Float32
+    # batchSize
+    batch_idx = randperm(len)[1:batchSize]
+    batch_X = X[batch_idx, :]
+    prediction = evalTreeArray(tree, batch_X)
+    if prediction === nothing
+        return 1f9
+    end
+    size_adjustment = 1f0
+    batch_y = y[batch_idx]
+    if weighted
+        batch_w = weights[batch_idx]
+        mse = MSE(prediction, batch_y, batch_w)
+        size_adjustment = 1f0 * len / batchSize
+    else
+        mse = MSE(prediction, batch_y)
+    end
+    return size_adjustment * mse / baselineMSE + countNodes(tree)*parsimony
+end

julia/Mutate.jl

@@ -0,0 +1,124 @@
+# Go through one mutation cycle
+function iterate(member::PopMember, T::Float32, curmaxsize::Integer, frequencyComplexity::Array{Float32, 1})::PopMember
+    prev = member.tree
+    tree = prev
+    #TODO - reconsider this
+    if batching
+        beforeLoss = scoreFuncBatch(prev)
+    else
+        beforeLoss = member.score
+    end
+
+    mutationChoice = rand()
+    #More constants => more likely to do constant mutation
+    weightAdjustmentMutateConstant = min(8, countConstants(prev))/8.0
+    cur_weights = copy(mutationWeights) .* 1.0
+    cur_weights[1] *= weightAdjustmentMutateConstant
+    n = countNodes(prev)
+    depth = countDepth(prev)
+
+    # If equation too big, don't add new operators
+    if n >= curmaxsize || depth >= maxdepth
+        cur_weights[3] = 0.0
+        cur_weights[4] = 0.0
+    end
+    cur_weights /= sum(cur_weights)
+    cweights = cumsum(cur_weights)
+
+    successful_mutation = false
+    #TODO: Currently we dont take this \/ into account
+    is_success_always_possible = true
+    attempts = 0
+    max_attempts = 10
+
+    #############################################
+    # Mutations
+    #############################################
+    while (!successful_mutation) && attempts < max_attempts
+        tree = copyNode(prev)
+        successful_mutation = true
+        if mutationChoice < cweights[1]
+            tree = mutateConstant(tree, T)
+
+            is_success_always_possible = true
+            # Mutating a constant shouldn't invalidate an already-valid function
+
+        elseif mutationChoice < cweights[2]
+            tree = mutateOperator(tree)
+
+            is_success_always_possible = true
+            # Can always mutate to the same operator
+
+        elseif mutationChoice < cweights[3]
+            if rand() < 0.5
+                tree = appendRandomOp(tree)
+            else
+                tree = prependRandomOp(tree)
+            end
+            is_success_always_possible = false
+            # Can potentially have a situation without success
+        elseif mutationChoice < cweights[4]
+            tree = insertRandomOp(tree)
+            is_success_always_possible = false
+        elseif mutationChoice < cweights[5]
+            tree = deleteRandomOp(tree)
+            is_success_always_possible = true
+        elseif mutationChoice < cweights[6]
+            tree = simplifyTree(tree) # Sometimes we simplify tree
+            tree = combineOperators(tree) # See if repeated constants at outer levels
+            return PopMember(tree, beforeLoss)
+
+            is_success_always_possible = true
+            # Simplification shouldn't hurt complexity; unless some non-symmetric constraint
+            # to commutative operator...
+
+        elseif mutationChoice < cweights[7]
+            tree = genRandomTree(5) # Sometimes we generate a new tree completely tree
+
+            is_success_always_possible = true
+        else # no mutation applied
+            return PopMember(tree, beforeLoss)
+        end
+
+        # Check for illegal equations
+        for i=1:nbin
+            if successful_mutation && flagBinOperatorComplexity(tree, i)
+                successful_mutation = false
+            end
+        end
+        for i=1:nuna
+            if successful_mutation && flagUnaOperatorComplexity(tree, i)
+                successful_mutation = false
+            end
+        end
+
+        attempts += 1
+    end
+    #############################################
+
+    if !successful_mutation
+        return PopMember(copyNode(prev), beforeLoss)
+    end
+
+    if batching
+        afterLoss = scoreFuncBatch(tree)
+    else
+        afterLoss = scoreFunc(tree)
+    end
+
+    if annealing
+        delta = afterLoss - beforeLoss
+        probChange = exp(-delta/(T*alpha))
+        if useFrequency
+            oldSize = countNodes(prev)
+            newSize = countNodes(tree)
+            probChange *= frequencyComplexity[oldSize] / frequencyComplexity[newSize]
+        end
+
+        return_unaltered = (isnan(afterLoss) || probChange < rand())
+        if return_unaltered
+            return PopMember(copyNode(prev), beforeLoss)
+        end
+    end
+    return PopMember(tree, afterLoss)
+end

julia/MutationFunctions.jl

@@ -0,0 +1,239 @@
+# Randomly convert an operator into another one (binary->binary;
+# unary->unary)
+function mutateOperator(tree::Node)::Node
+    if countOperators(tree) == 0
+        return tree
+    end
+    node = randomNode(tree)
+    while node.degree == 0
+        node = randomNode(tree)
+    end
+    if node.degree == 1
+        node.op = rand(1:length(unaops))
+    else
+        node.op = rand(1:length(binops))
+    end
+    return tree
+end
+
+# Randomly perturb a constant
+function mutateConstant(
+        tree::Node, T::Float32,
+        probNegate::Float32=0.01f0)::Node
+    # T is between 0 and 1.
+
+    if countConstants(tree) == 0
+        return tree
+    end
+    node = randomNode(tree)
+    while node.degree != 0 || node.constant == false
+        node = randomNode(tree)
+    end
+
+    bottom = 0.1f0
+    maxChange = perturbationFactor * T + 1.0f0 + bottom
+    factor = maxChange^Float32(rand())
+    makeConstBigger = rand() > 0.5
+
+    if makeConstBigger
+        node.val *= factor
+    else
+        node.val /= factor
+    end
+
+    if rand() > probNegate
+        node.val *= -1
+    end
+
+    return tree
+end
+
+# Add a random unary/binary operation to the end of a tree
+function appendRandomOp(tree::Node)::Node
+    node = randomNode(tree)
+    while node.degree != 0
+        node = randomNode(tree)
+    end
+
+    choice = rand()
+    makeNewBinOp = choice < nbin/nops
+    if rand() > 0.5
+        left = Float32(randn())
+    else
+        left = rand(1:nvar)
+    end
+    if rand() > 0.5
+        right = Float32(randn())
+    else
+        right = rand(1:nvar)
+    end
+
+    if makeNewBinOp
+        newnode = Node(
+            rand(1:length(binops)),
+            left,
+            right
+        )
+    else
+        newnode = Node(
+            rand(1:length(unaops)),
+            left
+        )
+    end
+    node.l = newnode.l
+    node.r = newnode.r
+    node.op = newnode.op
+    node.degree = newnode.degree
+    node.val = newnode.val
+    node.constant = newnode.constant
+    return tree
+end
+
+# Insert random node
+function insertRandomOp(tree::Node)::Node
+    node = randomNode(tree)
+    choice = rand()
+    makeNewBinOp = choice < nbin/nops
+    left = copyNode(node)
+
+    if makeNewBinOp
+        right = randomConstantNode()
+        newnode = Node(
+            rand(1:length(binops)),
+            left,
+            right
+        )
+    else
+        newnode = Node(
+            rand(1:length(unaops)),
+            left
+        )
+    end
+    node.l = newnode.l
+    node.r = newnode.r
+    node.op = newnode.op
+    node.degree = newnode.degree
+    node.val = newnode.val
+    node.constant = newnode.constant
+    return tree
+end
+
+# Add random node to the top of a tree
+function prependRandomOp(tree::Node)::Node
+    node = tree
+    choice = rand()
+    makeNewBinOp = choice < nbin/nops
+    left = copyNode(tree)
+
+    if makeNewBinOp
+        right = randomConstantNode()
+        newnode = Node(
+            rand(1:length(binops)),
+            left,
+            right
+        )
+    else
+        newnode = Node(
+            rand(1:length(unaops)),
+            left
+        )
+    end
+    node.l = newnode.l
+    node.r = newnode.r
+    node.op = newnode.op
+    node.degree = newnode.degree
+    node.val = newnode.val
+    node.constant = newnode.constant
+    return node
+end
+
+function randomConstantNode()::Node
+    if rand() > 0.5
+        val = Float32(randn())
+    else
+        val = rand(1:nvar)
+    end
+    newnode = Node(val)
+    return newnode
+end
+
+# Return a random node from the tree with parent
+function randomNodeAndParent(tree::Node, parent::Union{Node, Nothing})::Tuple{Node, Union{Node, Nothing}}
+    if tree.degree == 0
+        return tree, parent
+    end
+    a = countNodes(tree)
+    b = 0
+    c = 0
+    if tree.degree >= 1
+        b = countNodes(tree.l)
+    end
+    if tree.degree == 2
+        c = countNodes(tree.r)
+    end
+
+    i = rand(1:1+b+c)
+    if i <= b
+        return randomNodeAndParent(tree.l, tree)
+    elseif i == b + 1
+        return tree, parent
+    end
+
+    return randomNodeAndParent(tree.r, tree)
+end
+
+# Select a random node, and replace it an the subtree
+# with a variable or constant
+function deleteRandomOp(tree::Node)::Node
+    node, parent = randomNodeAndParent(tree, nothing)
+    isroot = (parent === nothing)
+
+    if node.degree == 0
+        # Replace with new constant
+        newnode = randomConstantNode()
+        node.l = newnode.l
+        node.r = newnode.r
+        node.op = newnode.op
+        node.degree = newnode.degree
+        node.val = newnode.val
+        node.constant = newnode.constant
+    elseif node.degree == 1
+        # Join one of the children with the parent
+        if isroot
+            return node.l
+        elseif parent.l == node
+            parent.l = node.l
+        else
+            parent.r = node.l
+        end
+    else
+        # Join one of the children with the parent
+        if rand() < 0.5
+            if isroot
+                return node.l
+            elseif parent.l == node
+                parent.l = node.l
+            else
+                parent.r = node.l
+            end
+        else
+            if isroot
+                return node.r
+            elseif parent.l == node
+                parent.l = node.r
+            else
+                parent.r = node.r
+            end
+        end
+    end
+    return tree
+end
+
+# Create a random equation by appending random operators
+function genRandomTree(length::Integer)::Node
+    tree = Node(1.0f0)
+    for i=1:length
+        tree = appendRandomOp(tree)
+    end
+    return tree
+end

julia/PopMember.jl

@@ -0,0 +1,10 @@
+# Define a member of population by equation, score, and age
+mutable struct PopMember
+    tree::Node
+    score::Float32
+    birth::Integer
+
+    PopMember(t::Node) = new(t, scoreFunc(t), getTime())
+    PopMember(t::Node, score::Float32) = new(t, score, getTime())
+
+end

julia/Population.jl

@@ -0,0 +1,40 @@
+# A list of members of the population, with easy constructors,
+#  which allow for random generation of new populations
+mutable struct Population
+    members::Array{PopMember, 1}
+    n::Integer
+
+    Population(pop::Array{PopMember, 1}) = new(pop, size(pop)[1])
+    Population(npop::Integer) = new([PopMember(genRandomTree(3)) for i=1:npop], npop)
+    Population(npop::Integer, nlength::Integer) = new([PopMember(genRandomTree(nlength)) for i=1:npop], npop)
+
+end
+
+# Sample 10 random members of the population, and make a new one
+function samplePop(pop::Population)::Population
+    idx = rand(1:pop.n, ns)
+    return Population(pop.members[idx])
+end
+
+# Sample the population, and get the best member from that sample
+function bestOfSample(pop::Population)::PopMember
+    sample = samplePop(pop)
+    best_idx = argmin([sample.members[member].score for member=1:sample.n])
+    return sample.members[best_idx]
+end
+
+function finalizeScores(pop::Population)::Population
+    need_recalculate = batching
+    if need_recalculate
+        @inbounds @simd for member=1:pop.n
+            pop.members[member].score = scoreFunc(pop.members[member].tree)
+        end
+    end
+    return pop
+end
+
+# Return best 10 examples
+function bestSubPop(pop::Population; topn::Integer=10)::Population
+    best_idx = sortperm([pop.members[member].score for member=1:pop.n])
+    return Population(pop.members[best_idx[1:topn]])
+end

julia/ProgramConstants.jl

@@ -0,0 +1,9 @@
+
+const maxdegree = 2
+const actualMaxsize = maxsize + maxdegree
+const len = size(X)[1]
+
+const nuna = size(unaops)[1]
+const nbin = size(binops)[1]
+const nops = nuna + nbin
+const nvar = size(X)[2];

julia/RegularizedEvolution.jl

@@ -0,0 +1,46 @@
+import Random: shuffle!
+
+# Pass through the population several times, replacing the oldest
+# with the fittest of a small subsample
+function regEvolCycle(pop::Population, T::Float32, curmaxsize::Integer,
+                      frequencyComplexity::Array{Float32, 1})::Population
+    # Batch over each subsample. Can give 15% improvement in speed; probably moreso for large pops.
+    # but is ultimately a different algorithm than regularized evolution, and might not be
+    # as good.
+    if fast_cycle
+        shuffle!(pop.members)
+        n_evol_cycles = round(Integer, pop.n/ns)
+        babies = Array{PopMember}(undef, n_evol_cycles)
+
+        # Iterate each ns-member sub-sample
+        @inbounds Threads.@threads for i=1:n_evol_cycles
+            best_score = Inf32
+            best_idx = 1+(i-1)*ns
+            # Calculate best member of the subsample:
+            for sub_i=1+(i-1)*ns:i*ns
+                if pop.members[sub_i].score < best_score
+                    best_score = pop.members[sub_i].score
+                    best_idx = sub_i
+                end
+            end
+            allstar = pop.members[best_idx]
+            babies[i] = iterate(allstar, T, curmaxsize, frequencyComplexity)
+        end
+
+        # Replace the n_evol_cycles-oldest members of each population
+        @inbounds for i=1:n_evol_cycles
+            oldest = argmin([pop.members[member].birth for member=1:pop.n])
+            pop.members[oldest] = babies[i]
+        end
+    else
+        for i=1:round(Integer, pop.n/ns)
+            allstar = bestOfSample(pop)
+            baby = iterate(allstar, T, curmaxsize, frequencyComplexity)
+            #printTree(baby.tree)
+            oldest = argmin([pop.members[member].birth for member=1:pop.n])
+            pop.members[oldest] = baby
+        end
+    end
+
+    return pop
+end

julia/SimplifyEquation.jl

@@ -0,0 +1,106 @@
+# Simplify tree
+function combineOperators(tree::Node)::Node
+    # NOTE: (const (+*-) const) already accounted for. Call simplifyTree before.
+    # ((const + var) + const) => (const + var)
+    # ((const * var) * const) => (const * var)
+    # ((const - var) - const) => (const - var)
+    # (want to add anything commutative!)
+    # TODO - need to combine plus/sub if they are both there.
+    if tree.degree == 0
+        return tree
+    elseif tree.degree == 1
+        tree.l = combineOperators(tree.l)
+    elseif tree.degree == 2
+        tree.l = combineOperators(tree.l)
+        tree.r = combineOperators(tree.r)
+    end
+
+    top_level_constant = tree.degree == 2 && (tree.l.constant || tree.r.constant)
+    if tree.degree == 2 && (binops[tree.op] === mult || binops[tree.op] === plus) && top_level_constant
+        op = tree.op
+        # Put the constant in r. Need to assume var in left for simplification assumption.
+        if tree.l.constant
+            tmp = tree.r
+            tree.r = tree.l
+            tree.l = tmp
+        end
+        topconstant = tree.r.val
+        # Simplify down first
+        below = tree.l
+        if below.degree == 2 && below.op == op
+            if below.l.constant
+                tree = below
+                tree.l.val = binops[op](tree.l.val, topconstant)
+            elseif below.r.constant
+                tree = below
+                tree.r.val = binops[op](tree.r.val, topconstant)
+            end
+        end
+    end
+
+    if tree.degree == 2 && binops[tree.op] === sub && top_level_constant
+        # Currently just simplifies subtraction. (can't assume both plus and sub are operators)
+        # Not commutative, so use different op.
+        if tree.l.constant
+            if tree.r.degree == 2 && binops[tree.r.op] === sub
+                if tree.r.l.constant
+                    #(const - (const - var)) => (var - const)
+                    l = tree.l
+                    r = tree.r
+                    simplified_const = -(l.val - r.l.val) #neg(sub(l.val, r.l.val))
+                    tree.l = tree.r.r
+                    tree.r = l
+                    tree.r.val = simplified_const
+                elseif tree.r.r.constant
+                    #(const - (var - const)) => (const - var)
+                    l = tree.l
+                    r = tree.r
+                    simplified_const = l.val + r.r.val #plus(l.val, r.r.val)
+                    tree.r = tree.r.l
+                    tree.l.val = simplified_const
+                end
+            end
+        else #tree.r.constant is true
+            if tree.l.degree == 2 && binops[tree.l.op] === sub
+                if tree.l.l.constant
+                    #((const - var) - const) => (const - var)
+                    l = tree.l
+                    r = tree.r
+                    simplified_const = l.l.val - r.val#sub(l.l.val, r.val)
+                    tree.r = tree.l.r
+                    tree.l = r
+                    tree.l.val = simplified_const
+                elseif tree.l.r.constant
+                    #((var - const) - const) => (var - const)
+                    l = tree.l
+                    r = tree.r
+                    simplified_const = r.val + l.r.val #plus(r.val, l.r.val)
+                    tree.l = tree.l.l
+                    tree.r.val = simplified_const
+                end
+            end
+        end
+    end
+    return tree
+end
+
+# Simplify tree
+function simplifyTree(tree::Node)::Node
+    if tree.degree == 1
+        tree.l = simplifyTree(tree.l)
+        if tree.l.degree == 0 && tree.l.constant
+            return Node(unaops[tree.op](tree.l.val))
+        end
+    elseif tree.degree == 2
+        tree.l = simplifyTree(tree.l)
+        tree.r = simplifyTree(tree.r)
+        constantsBelow = (
+             tree.l.degree == 0 && tree.l.constant &&
+             tree.r.degree == 0 && tree.r.constant
+        )
+        if constantsBelow
+            return Node(binops[tree.op](tree.l.val, tree.r.val))
+        end
+    end
+    return tree
+end

julia/SingleIteration.jl

@@ -0,0 +1,28 @@
+# Cycle through regularized evolution many times,
+# printing the fittest equation every 10% through
+function run(
+        pop::Population,
+        ncycles::Integer,
+        curmaxsize::Integer,
+        frequencyComplexity::Array{Float32, 1};
+        verbosity::Integer=0
+       )::Population
+
+    allT = LinRange(1.0f0, 0.0f0, ncycles)
+    for iT in 1:size(allT)[1]
+        if annealing
+            pop = regEvolCycle(pop, allT[iT], curmaxsize, frequencyComplexity)
+        else
+            pop = regEvolCycle(pop, 1.0f0, curmaxsize, frequencyComplexity)
+        end
+
+        if verbosity > 0 && (iT % verbosity == 0)
+            bestPops = bestSubPop(pop)
+            bestCurScoreIdx = argmin([bestPops.members[member].score for member=1:bestPops.n])
+            bestCurScore = bestPops.members[bestCurScoreIdx].score
+            debug(verbosity, bestCurScore, " is the score for ", stringTree(bestPops.members[bestCurScoreIdx].tree))
+        end
+    end
+
+    return pop
+end

julia/Utils.jl

@@ -0,0 +1,34 @@
+import Printf: @printf
+
+function id(x::Float32)::Float32
+    x
+end
+
+function debug(verbosity, string...)
+    verbosity > 0 ? println(string...) : nothing
+end
+
+function getTime()::Integer
+    return round(Integer, 1e3*(time()-1.6e9))
+end
+
+# Check for errors before they happen
+function testConfiguration()
+    test_input = LinRange(-100f0, 100f0, 99)
+
+    try
+        for left in test_input
+            for right in test_input
+                for binop in binops
+                    test_output = binop.(left, right)
+                end
+            end
+            for unaop in unaops
+                test_output = unaop.(left)
+            end
+        end
+    catch error
+        @printf("\n\nYour configuration is invalid - one of your operators is not well-defined over the real line.\n\n\n")
+        throw(error)
+    end
+end

julia/halloffame.jl

@@ -0,0 +1,8 @@
+# List of the best members seen all time
+mutable struct HallOfFame
+    members::Array{PopMember, 1}
+    exists::Array{Bool, 1} #Whether it has been set
+
+    # Arranged by complexity - store one at each.
+    HallOfFame() = new([PopMember(Node(1f0), 1f9) for i=1:actualMaxsize], [false for i=1:actualMaxsize])
+end

julia/sr.jl

@@ -1,1057 +1,4 @@
-import Optim
 import Printf: @printf
-import Random: shuffle!, randperm
-
-const maxdegree = 2
-const actualMaxsize = maxsize + maxdegree
-
-
-# Sum of square error between two arrays
-function SSE(x::Array{Float32}, y::Array{Float32})::Float32
-    diff = (x - y)
-    return sum(diff .* diff)
-end
-function SSE(x::Nothing, y::Array{Float32})::Float32
-    return 1f9
-end
-
-# Sum of square error between two arrays, with weights
-function SSE(x::Array{Float32}, y::Array{Float32}, w::Array{Float32})::Float32
-    diff = (x - y)
-    return sum(diff .* diff .* w)
-end
-function SSE(x::Nothing, y::Array{Float32}, w::Array{Float32})::Float32
-    return Nothing
-end
-
-# Mean of square error between two arrays
-function MSE(x::Nothing, y::Array{Float32})::Float32
-    return 1f9
-end
-
-# Mean of square error between two arrays
-function MSE(x::Array{Float32}, y::Array{Float32})::Float32
-    return SSE(x, y)/size(x)[1]
-end
-
-# Mean of square error between two arrays
-function MSE(x::Nothing, y::Array{Float32}, w::Array{Float32})::Float32
-    return 1f9
-end
-
-# Mean of square error between two arrays
-function MSE(x::Array{Float32}, y::Array{Float32}, w::Array{Float32})::Float32
-    return SSE(x, y, w)/sum(w)
-end
-
-const len = size(X)[1]
-
-if weighted
-    const avgy = sum(y .* weights)/sum(weights)
-    const baselineMSE = MSE(y, convert(Array{Float32, 1}, ones(len) .* avgy), weights)
-else
-    const avgy = sum(y)/len
-    const baselineMSE = MSE(y, convert(Array{Float32, 1}, ones(len) .* avgy))
-end
-
-
-function id(x::Float32)::Float32
-    x
-end
-
-const nuna = size(unaops)[1]
-const nbin = size(binops)[1]
-const nops = nuna + nbin
-const nvar = size(X)[2];
-
-function debug(verbosity, string...)
-    verbosity > 0 ? println(string...) : nothing
-end
-
-function getTime()::Integer
-    return round(Integer, 1e3*(time()-1.6e9))
-end
-
-# Define a serialization format for the symbolic equations:
-mutable struct Node
-    #Holds operators, variables, constants in a tree
-    degree::Integer #0 for constant/variable, 1 for cos/sin, 2 for +/* etc.
-    val::Union{Float32, Integer} #Either const value, or enumerates variable
-    constant::Bool #false if variable
-    op::Integer #enumerates operator (separately for degree=1,2)
-    l::Union{Node, Nothing}
-    r::Union{Node, Nothing}
-
-    Node(val::Float32) = new(0, val, true, 1, nothing, nothing)
-    Node(val::Integer) = new(0, val, false, 1, nothing, nothing)
-    Node(op::Integer, l::Node) = new(1, 0.0f0, false, op, l, nothing)
-    Node(op::Integer, l::Union{Float32, Integer}) = new(1, 0.0f0, false, op, Node(l), nothing)
-    Node(op::Integer, l::Node, r::Node) = new(2, 0.0f0, false, op, l, r)
-
-    #Allow to pass the leaf value without additional node call:
-    Node(op::Integer, l::Union{Float32, Integer}, r::Node) = new(2, 0.0f0, false, op, Node(l), r)
-    Node(op::Integer, l::Node, r::Union{Float32, Integer}) = new(2, 0.0f0, false, op, l, Node(r))
-    Node(op::Integer, l::Union{Float32, Integer}, r::Union{Float32, Integer}) = new(2, 0.0f0, false, op, Node(l), Node(r))
-end
-
-# Copy an equation (faster than deepcopy)
-function copyNode(tree::Node)::Node
-   if tree.degree == 0
-       return Node(tree.val)
-   elseif tree.degree == 1
-       return Node(tree.op, copyNode(tree.l))
-    else
-        return Node(tree.op, copyNode(tree.l), copyNode(tree.r))
-   end
-end
-
-# Count the operators, constants, variables in an equation
-function countNodes(tree::Node)::Integer
-    if tree.degree == 0
-        return 1
-    elseif tree.degree == 1
-        return 1 + countNodes(tree.l)
-    else
-        return 1 + countNodes(tree.l) + countNodes(tree.r)
-    end
-end
-
-# Count the max depth of a tree
-function countDepth(tree::Node)::Integer
-    if tree.degree == 0
-        return 1
-    elseif tree.degree == 1
-        return 1 + countDepth(tree.l)
-    else
-        return 1 + max(countDepth(tree.l), countDepth(tree.r))
-    end
-end
-
-# Convert an equation to a string
-function stringTree(tree::Node)::String
-    if tree.degree == 0
-        if tree.constant
-            return string(tree.val)
-        else
-            if useVarMap
-                return varMap[tree.val]
-            else
-                return "x$(tree.val - 1)"
-            end
-        end
-    elseif tree.degree == 1
-        return "$(unaops[tree.op])($(stringTree(tree.l)))"
-    else
-        return "$(binops[tree.op])($(stringTree(tree.l)), $(stringTree(tree.r)))"
-    end
-end
-
-# Print an equation
-function printTree(tree::Node)
-    println(stringTree(tree))
-end
-
-# Return a random node from the tree
-function randomNode(tree::Node)::Node
-    if tree.degree == 0
-        return tree
-    end
-    a = countNodes(tree)
-    b = 0
-    c = 0
-    if tree.degree >= 1
-        b = countNodes(tree.l)
-    end
-    if tree.degree == 2
-        c = countNodes(tree.r)
-    end
-
-    i = rand(1:1+b+c)
-    if i <= b
-        return randomNode(tree.l)
-    elseif i == b + 1
-        return tree
-    end
-
-    return randomNode(tree.r)
-end
-
-# Count the number of unary operators in the equation
-function countUnaryOperators(tree::Node)::Integer
-    if tree.degree == 0
-        return 0
-    elseif tree.degree == 1
-        return 1 + countUnaryOperators(tree.l)
-    else
-        return 0 + countUnaryOperators(tree.l) + countUnaryOperators(tree.r)
-    end
-end
-
-# Count the number of binary operators in the equation
-function countBinaryOperators(tree::Node)::Integer
-    if tree.degree == 0
-        return 0
-    elseif tree.degree == 1
-        return 0 + countBinaryOperators(tree.l)
-    else
-        return 1 + countBinaryOperators(tree.l) + countBinaryOperators(tree.r)
-    end
-end
-
-# Count the number of operators in the equation
-function countOperators(tree::Node)::Integer
-    return countUnaryOperators(tree) + countBinaryOperators(tree)
-end
-
-# Randomly convert an operator into another one (binary->binary;
-# unary->unary)
-function mutateOperator(tree::Node)::Node
-    if countOperators(tree) == 0
-        return tree
-    end
-    node = randomNode(tree)
-    while node.degree == 0
-        node = randomNode(tree)
-    end
-    if node.degree == 1
-        node.op = rand(1:length(unaops))
-    else
-        node.op = rand(1:length(binops))
-    end
-    return tree
-end
-
-# Count the number of constants in an equation
-function countConstants(tree::Node)::Integer
-    if tree.degree == 0
-        return convert(Integer, tree.constant)
-    elseif tree.degree == 1
-        return 0 + countConstants(tree.l)
-    else
-        return 0 + countConstants(tree.l) + countConstants(tree.r)
-    end
-end
-
-# Randomly perturb a constant
-function mutateConstant(
-        tree::Node, T::Float32,
-        probNegate::Float32=0.01f0)::Node
-    # T is between 0 and 1.
-
-    if countConstants(tree) == 0
-        return tree
-    end
-    node = randomNode(tree)
-    while node.degree != 0 || node.constant == false
-        node = randomNode(tree)
-    end
-
-    bottom = 0.1f0
-    maxChange = perturbationFactor * T + 1.0f0 + bottom
-    factor = maxChange^Float32(rand())
-    makeConstBigger = rand() > 0.5
-
-    if makeConstBigger
-        node.val *= factor
-    else
-        node.val /= factor
-    end
-
-    if rand() > probNegate
-        node.val *= -1
-    end
-
-    return tree
-end
-
-# Evaluate an equation over an array of datapoints
-function evalTreeArray(tree::Node)::Union{Array{Float32, 1}, Nothing}
-    return evalTreeArray(tree, X)
-end
-
-
-# Evaluate an equation over an array of datapoints
-function evalTreeArray(tree::Node, cX::Array{Float32, 2})::Union{Array{Float32, 1}, Nothing}
-    clen = size(cX)[1]
-    if tree.degree == 0
-        if tree.constant
-            return fill(tree.val, clen)
-        else
-            return copy(cX[:, tree.val])
-        end
-    elseif tree.degree == 1
-        cumulator = evalTreeArray(tree.l, cX)
-        if cumulator === nothing
-            return nothing
-        end
-        op_idx = tree.op
-        UNAOP!(cumulator, op_idx, clen)
-        @inbounds for i=1:clen
-            if isinf(cumulator[i]) || isnan(cumulator[i])
-                return nothing
-            end
-        end
-        return cumulator
-    else
-        cumulator = evalTreeArray(tree.l, cX)
-        if cumulator === nothing
-            return nothing
-        end
-        array2 = evalTreeArray(tree.r, cX)
-        if array2 === nothing
-            return nothing
-        end
-        op_idx = tree.op
-        BINOP!(cumulator, array2, op_idx, clen)
-        @inbounds for i=1:clen
-            if isinf(cumulator[i]) || isnan(cumulator[i])
-                return nothing
-            end
-        end
-        return cumulator
-    end
-end
-
-# Score an equation
-function scoreFunc(tree::Node)::Float32
-    prediction = evalTreeArray(tree)
-    if prediction === nothing
-        return 1f9
-    end
-    if weighted
-        mse = MSE(prediction, y, weights)
-    else
-        mse = MSE(prediction, y)
-    end
-    return mse / baselineMSE + countNodes(tree)*parsimony
-end
-
-# Score an equation with a small batch
-function scoreFuncBatch(tree::Node)::Float32
-    # batchSize
-    batch_idx = randperm(len)[1:batchSize]
-    batch_X = X[batch_idx, :]
-    prediction = evalTreeArray(tree, batch_X)
-    if prediction === nothing
-        return 1f9
-    end
-    size_adjustment = 1f0
-    batch_y = y[batch_idx]
-    if weighted
-        batch_w = weights[batch_idx]
-        mse = MSE(prediction, batch_y, batch_w)
-        size_adjustment = 1f0 * len / batchSize
-    else
-        mse = MSE(prediction, batch_y)
-    end
-    return size_adjustment * mse / baselineMSE + countNodes(tree)*parsimony
-end
-
-# Add a random unary/binary operation to the end of a tree
-function appendRandomOp(tree::Node)::Node
-    node = randomNode(tree)
-    while node.degree != 0
-        node = randomNode(tree)
-    end
-
-    choice = rand()
-    makeNewBinOp = choice < nbin/nops
-    if rand() > 0.5
-        left = Float32(randn())
-    else
-        left = rand(1:nvar)
-    end
-    if rand() > 0.5
-        right = Float32(randn())
-    else
-        right = rand(1:nvar)
-    end
-
-    if makeNewBinOp
-        newnode = Node(
-            rand(1:length(binops)),
-            left,
-            right
-        )
-    else
-        newnode = Node(
-            rand(1:length(unaops)),
-            left
-        )
-    end
-    node.l = newnode.l
-    node.r = newnode.r
-    node.op = newnode.op
-    node.degree = newnode.degree
-    node.val = newnode.val
-    node.constant = newnode.constant
-    return tree
-end
-
-# Insert random node
-function insertRandomOp(tree::Node)::Node
-    node = randomNode(tree)
-    choice = rand()
-    makeNewBinOp = choice < nbin/nops
-    left = copyNode(node)
-
-    if makeNewBinOp
-        right = randomConstantNode()
-        newnode = Node(
-            rand(1:length(binops)),
-            left,
-            right
-        )
-    else
-        newnode = Node(
-            rand(1:length(unaops)),
-            left
-        )
-    end
-    node.l = newnode.l
-    node.r = newnode.r
-    node.op = newnode.op
-    node.degree = newnode.degree
-    node.val = newnode.val
-    node.constant = newnode.constant
-    return tree
-end
-
-# Add random node to the top of a tree
-function prependRandomOp(tree::Node)::Node
-    node = tree
-    choice = rand()
-    makeNewBinOp = choice < nbin/nops
-    left = copyNode(tree)
-
-    if makeNewBinOp
-        right = randomConstantNode()
-        newnode = Node(
-            rand(1:length(binops)),
-            left,
-            right
-        )
-    else
-        newnode = Node(
-            rand(1:length(unaops)),
-            left
-        )
-    end
-    node.l = newnode.l
-    node.r = newnode.r
-    node.op = newnode.op
-    node.degree = newnode.degree
-    node.val = newnode.val
-    node.constant = newnode.constant
-    return node
-end
-
-function randomConstantNode()::Node
-    if rand() > 0.5
-        val = Float32(randn())
-    else
-        val = rand(1:nvar)
-    end
-    newnode = Node(val)
-    return newnode
-end
-
-# Return a random node from the tree with parent
-function randomNodeAndParent(tree::Node, parent::Union{Node, Nothing})::Tuple{Node, Union{Node, Nothing}}
-    if tree.degree == 0
-        return tree, parent
-    end
-    a = countNodes(tree)
-    b = 0
-    c = 0
-    if tree.degree >= 1
-        b = countNodes(tree.l)
-    end
-    if tree.degree == 2
-        c = countNodes(tree.r)
-    end
-
-    i = rand(1:1+b+c)
-    if i <= b
-        return randomNodeAndParent(tree.l, tree)
-    elseif i == b + 1
-        return tree, parent
-    end
-
-    return randomNodeAndParent(tree.r, tree)
-end
-
-# Select a random node, and replace it an the subtree
-# with a variable or constant
-function deleteRandomOp(tree::Node)::Node
-    node, parent = randomNodeAndParent(tree, nothing)
-    isroot = (parent === nothing)
-
-    if node.degree == 0
-        # Replace with new constant
-        newnode = randomConstantNode()
-        node.l = newnode.l
-        node.r = newnode.r
-        node.op = newnode.op
-        node.degree = newnode.degree
-        node.val = newnode.val
-        node.constant = newnode.constant
-    elseif node.degree == 1
-        # Join one of the children with the parent
-        if isroot
-            return node.l
-        elseif parent.l == node
-            parent.l = node.l
-        else
-            parent.r = node.l
-        end
-    else
-        # Join one of the children with the parent
-        if rand() < 0.5
-            if isroot
-                return node.l
-            elseif parent.l == node
-                parent.l = node.l
-            else
-                parent.r = node.l
-            end
-        else
-            if isroot
-                return node.r
-            elseif parent.l == node
-                parent.l = node.r
-            else
-                parent.r = node.r
-            end
-        end
-    end
-    return tree
-end
-
-# Simplify tree
-function combineOperators(tree::Node)::Node
-    # NOTE: (const (+*-) const) already accounted for. Call simplifyTree before.
-    # ((const + var) + const) => (const + var)
-    # ((const * var) * const) => (const * var)
-    # ((const - var) - const) => (const - var)
-    # (want to add anything commutative!)
-    # TODO - need to combine plus/sub if they are both there.
-    if tree.degree == 0
-        return tree
-    elseif tree.degree == 1
-        tree.l = combineOperators(tree.l)
-    elseif tree.degree == 2
-        tree.l = combineOperators(tree.l)
-        tree.r = combineOperators(tree.r)
-    end
-
-    top_level_constant = tree.degree == 2 && (tree.l.constant || tree.r.constant)
-    if tree.degree == 2 && (binops[tree.op] === mult || binops[tree.op] === plus) && top_level_constant
-        op = tree.op
-        # Put the constant in r. Need to assume var in left for simplification assumption.
-        if tree.l.constant
-            tmp = tree.r
-            tree.r = tree.l
-            tree.l = tmp
-        end
-        topconstant = tree.r.val
-        # Simplify down first
-        below = tree.l
-        if below.degree == 2 && below.op == op
-            if below.l.constant
-                tree = below
-                tree.l.val = binops[op](tree.l.val, topconstant)
-            elseif below.r.constant
-                tree = below
-                tree.r.val = binops[op](tree.r.val, topconstant)
-            end
-        end
-    end
-
-    if tree.degree == 2 && binops[tree.op] === sub && top_level_constant
-        # Currently just simplifies subtraction. (can't assume both plus and sub are operators)
-        # Not commutative, so use different op.
-        if tree.l.constant
-            if tree.r.degree == 2 && binops[tree.r.op] === sub
-                if tree.r.l.constant
-                    #(const - (const - var)) => (var - const)
-                    l = tree.l
-                    r = tree.r
-                    simplified_const = -(l.val - r.l.val) #neg(sub(l.val, r.l.val))
-                    tree.l = tree.r.r
-                    tree.r = l
-                    tree.r.val = simplified_const
-                elseif tree.r.r.constant
-                    #(const - (var - const)) => (const - var)
-                    l = tree.l
-                    r = tree.r
-                    simplified_const = l.val + r.r.val #plus(l.val, r.r.val)
-                    tree.r = tree.r.l
-                    tree.l.val = simplified_const
-                end
-            end
-        else #tree.r.constant is true
-            if tree.l.degree == 2 && binops[tree.l.op] === sub
-                if tree.l.l.constant
-                    #((const - var) - const) => (const - var)
-                    l = tree.l
-                    r = tree.r
-                    simplified_const = l.l.val - r.val#sub(l.l.val, r.val)
-                    tree.r = tree.l.r
-                    tree.l = r
-                    tree.l.val = simplified_const
-                elseif tree.l.r.constant
-                    #((var - const) - const) => (var - const)
-                    l = tree.l
-                    r = tree.r
-                    simplified_const = r.val + l.r.val #plus(r.val, l.r.val)
-                    tree.l = tree.l.l
-                    tree.r.val = simplified_const
-                end
-            end
-        end
-    end
-    return tree
-end
-
-# Simplify tree
-function simplifyTree(tree::Node)::Node
-    if tree.degree == 1
-        tree.l = simplifyTree(tree.l)
-        if tree.l.degree == 0 && tree.l.constant
-            return Node(unaops[tree.op](tree.l.val))
-        end
-    elseif tree.degree == 2
-        tree.l = simplifyTree(tree.l)
-        tree.r = simplifyTree(tree.r)
-        constantsBelow = (
-             tree.l.degree == 0 && tree.l.constant &&
-             tree.r.degree == 0 && tree.r.constant
-        )
-        if constantsBelow
-            return Node(binops[tree.op](tree.l.val, tree.r.val))
-        end
-    end
-    return tree
-end
-
-# Define a member of population by equation, score, and age
-mutable struct PopMember
-    tree::Node
-    score::Float32
-    birth::Integer
-
-    PopMember(t::Node) = new(t, scoreFunc(t), getTime())
-    PopMember(t::Node, score::Float32) = new(t, score, getTime())
-
-end
-
-# Check if any binary operator are overly complex
-function flagBinOperatorComplexity(tree::Node, op::Int)::Bool
-    if tree.degree == 0
-        return false
-    elseif tree.degree == 1
-        return flagBinOperatorComplexity(tree.l, op)
-    else
-        if tree.op == op
-            overly_complex = (
-                    ((bin_constraints[op][1] > -1) &&
-                     (countNodes(tree.l) > bin_constraints[op][1]))
-                      ||
-                    ((bin_constraints[op][2] > -1) &&
-                     (countNodes(tree.r) > bin_constraints[op][2]))
-                )
-            if overly_complex
-                return true
-            end
-        end
-        return (flagBinOperatorComplexity(tree.l, op) || flagBinOperatorComplexity(tree.r, op))
-    end
-end
-
-# Check if any unary operators are overly complex
-function flagUnaOperatorComplexity(tree::Node, op::Int)::Bool
-    if tree.degree == 0
-        return false
-    elseif tree.degree == 1
-        if tree.op == op
-            overly_complex = (
-                      (una_constraints[op] > -1) &&
-                      (countNodes(tree.l) > una_constraints[op])
-                )
-            if overly_complex
-                return true
-            end
-        end
-        return flagUnaOperatorComplexity(tree.l, op)
-    else
-        return (flagUnaOperatorComplexity(tree.l, op) || flagUnaOperatorComplexity(tree.r, op))
-    end
-end
-
-# Go through one simulated annealing mutation cycle
-#  exp(-delta/T) defines probability of accepting a change
-function iterate(member::PopMember, T::Float32, curmaxsize::Integer, frequencyComplexity::Array{Float32, 1})::PopMember
-    prev = member.tree
-    tree = prev
-    #TODO - reconsider this
-    if batching
-        beforeLoss = scoreFuncBatch(prev)
-    else
-        beforeLoss = member.score
-    end
-
-    mutationChoice = rand()
-    #More constants => more likely to do constant mutation
-    weightAdjustmentMutateConstant = min(8, countConstants(prev))/8.0
-    cur_weights = copy(mutationWeights) .* 1.0
-    cur_weights[1] *= weightAdjustmentMutateConstant
-    n = countNodes(prev)
-    depth = countDepth(prev)
-
-    # If equation too big, don't add new operators
-    if n >= curmaxsize || depth >= maxdepth
-        cur_weights[3] = 0.0
-        cur_weights[4] = 0.0
-    end
-    cur_weights /= sum(cur_weights)
-    cweights = cumsum(cur_weights)
-
-    successful_mutation = false
-    #TODO: Currently we dont take this \/ into account
-    is_success_always_possible = true
-    attempts = 0
-    max_attempts = 10
-    
-    #############################################
-    # Mutations
-    #############################################
-    while (!successful_mutation) && attempts < max_attempts
-        tree = copyNode(prev)
-        successful_mutation = true
-        if mutationChoice < cweights[1]
-            tree = mutateConstant(tree, T)
-
-            is_success_always_possible = true
-            # Mutating a constant shouldn't invalidate an already-valid function
-
-        elseif mutationChoice < cweights[2]
-            tree = mutateOperator(tree)
-
-            is_success_always_possible = true
-            # Can always mutate to the same operator
-
-        elseif mutationChoice < cweights[3]
-            if rand() < 0.5
-                tree = appendRandomOp(tree)
-            else
-                tree = prependRandomOp(tree)
-            end
-            is_success_always_possible = false
-            # Can potentially have a situation without success
-        elseif mutationChoice < cweights[4]
-            tree = insertRandomOp(tree)
-            is_success_always_possible = false
-        elseif mutationChoice < cweights[5]
-            tree = deleteRandomOp(tree)
-            is_success_always_possible = true
-        elseif mutationChoice < cweights[6]
-            tree = simplifyTree(tree) # Sometimes we simplify tree
-            tree = combineOperators(tree) # See if repeated constants at outer levels
-            return PopMember(tree, beforeLoss)
-
-            is_success_always_possible = true
-            # Simplification shouldn't hurt complexity; unless some non-symmetric constraint
-            # to commutative operator...
-
-        elseif mutationChoice < cweights[7]
-            tree = genRandomTree(5) # Sometimes we generate a new tree completely tree
-
-            is_success_always_possible = true
-        else # no mutation applied
-            return PopMember(tree, beforeLoss)
-        end
-
-        # Check for illegal equations
-        for i=1:nbin
-            if successful_mutation && flagBinOperatorComplexity(tree, i)
-                successful_mutation = false
-            end
-        end
-        for i=1:nuna
-            if successful_mutation && flagUnaOperatorComplexity(tree, i)
-                successful_mutation = false
-            end
-        end
-
-        attempts += 1
-    end
-    #############################################
-
-    if !successful_mutation
-        return PopMember(copyNode(prev), beforeLoss)
-    end
-
-    if batching
-        afterLoss = scoreFuncBatch(tree)
-    else
-        afterLoss = scoreFunc(tree)
-    end
-
-    if annealing
-        delta = afterLoss - beforeLoss
-        probChange = exp(-delta/(T*alpha))
-        if useFrequency
-            oldSize = countNodes(prev)
-            newSize = countNodes(tree)
-            probChange *= frequencyComplexity[oldSize] / frequencyComplexity[newSize]
-        end
-
-        return_unaltered = (isnan(afterLoss) || probChange < rand())
-        if return_unaltered
-            return PopMember(copyNode(prev), beforeLoss)
-        end
-    end
-    return PopMember(tree, afterLoss)
-end
-
-# Create a random equation by appending random operators
-function genRandomTree(length::Integer)::Node
-    tree = Node(1.0f0)
-    for i=1:length
-        tree = appendRandomOp(tree)
-    end
-    return tree
-end
-
-
-# A list of members of the population, with easy constructors,
-#  which allow for random generation of new populations
-mutable struct Population
-    members::Array{PopMember, 1}
-    n::Integer
-
-    Population(pop::Array{PopMember, 1}) = new(pop, size(pop)[1])
-    Population(npop::Integer) = new([PopMember(genRandomTree(3)) for i=1:npop], npop)
-    Population(npop::Integer, nlength::Integer) = new([PopMember(genRandomTree(nlength)) for i=1:npop], npop)
-
-end
-
-# Sample 10 random members of the population, and make a new one
-function samplePop(pop::Population)::Population
-    idx = rand(1:pop.n, ns)
-    return Population(pop.members[idx])
-end
-
-# Sample the population, and get the best member from that sample
-function bestOfSample(pop::Population)::PopMember
-    sample = samplePop(pop)
-    best_idx = argmin([sample.members[member].score for member=1:sample.n])
-    return sample.members[best_idx]
-end
-
-function finalizeScores(pop::Population)::Population
-    need_recalculate = batching
-    if need_recalculate
-        @inbounds @simd for member=1:pop.n
-            pop.members[member].score = scoreFunc(pop.members[member].tree)
-        end
-    end
-    return pop
-end
-
-# Return best 10 examples
-function bestSubPop(pop::Population; topn::Integer=10)::Population
-    best_idx = sortperm([pop.members[member].score for member=1:pop.n])
-    return Population(pop.members[best_idx[1:topn]])
-end
-
-# Pass through the population several times, replacing the oldest
-# with the fittest of a small subsample
-function regEvolCycle(pop::Population, T::Float32, curmaxsize::Integer,
-                      frequencyComplexity::Array{Float32, 1})::Population
-    # Batch over each subsample. Can give 15% improvement in speed; probably moreso for large pops.
-    # but is ultimately a different algorithm than regularized evolution, and might not be
-    # as good.
-    if fast_cycle
-        shuffle!(pop.members)
-        n_evol_cycles = round(Integer, pop.n/ns)
-        babies = Array{PopMember}(undef, n_evol_cycles)
-
-        # Iterate each ns-member sub-sample
-        @inbounds Threads.@threads for i=1:n_evol_cycles
-            best_score = Inf32
-            best_idx = 1+(i-1)*ns
-            # Calculate best member of the subsample:
-            for sub_i=1+(i-1)*ns:i*ns
-                if pop.members[sub_i].score < best_score
-                    best_score = pop.members[sub_i].score
-                    best_idx = sub_i
-                end
-            end
-            allstar = pop.members[best_idx]
-            babies[i] = iterate(allstar, T, curmaxsize, frequencyComplexity)
-        end
-
-        # Replace the n_evol_cycles-oldest members of each population
-        @inbounds for i=1:n_evol_cycles
-            oldest = argmin([pop.members[member].birth for member=1:pop.n])
-            pop.members[oldest] = babies[i]
-        end
-    else
-        for i=1:round(Integer, pop.n/ns)
-            allstar = bestOfSample(pop)
-            baby = iterate(allstar, T, curmaxsize, frequencyComplexity)
-            #printTree(baby.tree)
-            oldest = argmin([pop.members[member].birth for member=1:pop.n])
-            pop.members[oldest] = baby
-        end
-    end
-
-    return pop
-end
-
-# Cycle through regularized evolution many times,
-# printing the fittest equation every 10% through
-function run(
-        pop::Population,
-        ncycles::Integer,
-        curmaxsize::Integer,
-        frequencyComplexity::Array{Float32, 1};
-        verbosity::Integer=0
-       )::Population
-
-    allT = LinRange(1.0f0, 0.0f0, ncycles)
-    for iT in 1:size(allT)[1]
-        if annealing
-            pop = regEvolCycle(pop, allT[iT], curmaxsize, frequencyComplexity)
-        else
-            pop = regEvolCycle(pop, 1.0f0, curmaxsize, frequencyComplexity)
-        end
-
-        if verbosity > 0 && (iT % verbosity == 0)
-            bestPops = bestSubPop(pop)
-            bestCurScoreIdx = argmin([bestPops.members[member].score for member=1:bestPops.n])
-            bestCurScore = bestPops.members[bestCurScoreIdx].score
-            debug(verbosity, bestCurScore, " is the score for ", stringTree(bestPops.members[bestCurScoreIdx].tree))
-        end
-    end
-
-    return pop
-end
-
-# Get all the constants from a tree
-function getConstants(tree::Node)::Array{Float32, 1}
-    if tree.degree == 0
-        if tree.constant
-            return [tree.val]
-        else
-            return Float32[]
-        end
-    elseif tree.degree == 1
-        return getConstants(tree.l)
-    else
-        both = [getConstants(tree.l), getConstants(tree.r)]
-        return [constant for subtree in both for constant in subtree]
-    end
-end
-
-# Set all the constants inside a tree
-function setConstants(tree::Node, constants::Array{Float32, 1})
-    if tree.degree == 0
-        if tree.constant
-            tree.val = constants[1]
-        end
-    elseif tree.degree == 1
-        setConstants(tree.l, constants)
-    else
-        numberLeft = countConstants(tree.l)
-        setConstants(tree.l, constants)
-        setConstants(tree.r, constants[numberLeft+1:end])
-    end
-end
-
-
-# Proxy function for optimization
-function optFunc(x::Array{Float32, 1}, tree::Node)::Float32
-    setConstants(tree, x)
-    return scoreFunc(tree)
-end
-
-# Use Nelder-Mead to optimize the constants in an equation
-function optimizeConstants(member::PopMember)::PopMember
-    nconst = countConstants(member.tree)
-    if nconst == 0
-        return member
-    end
-    x0 = getConstants(member.tree)
-    f(x::Array{Float32,1})::Float32 = optFunc(x, member.tree)
-    if size(x0)[1] == 1
-        algorithm = Optim.Newton
-    else
-        algorithm = Optim.NelderMead
-    end
-
-    try
-        result = Optim.optimize(f, x0, algorithm(), Optim.Options(iterations=100))
-        # Try other initial conditions:
-        for i=1:nrestarts
-            tmpresult = Optim.optimize(f, x0 .* (1f0 .+ 5f-1*randn(Float32, size(x0)[1])), algorithm(), Optim.Options(iterations=100))
-            if tmpresult.minimum < result.minimum
-                result = tmpresult
-            end
-        end
-
-        if Optim.converged(result)
-            setConstants(member.tree, result.minimizer)
-            member.score = convert(Float32, result.minimum)
-            member.birth = getTime()
-        else
-            setConstants(member.tree, x0)
-        end
-    catch error
-        # Fine if optimization encountered domain error, just return x0
-        if isa(error, AssertionError)
-            setConstants(member.tree, x0)
-        else
-            throw(error)
-        end
-    end
-    return member
-end
-
-
-# List of the best members seen all time
-mutable struct HallOfFame
-    members::Array{PopMember, 1}
-    exists::Array{Bool, 1} #Whether it has been set
-
-    # Arranged by complexity - store one at each.
-    HallOfFame() = new([PopMember(Node(1f0), 1f9) for i=1:actualMaxsize], [false for i=1:actualMaxsize])
-end
-
-
-# Check for errors before they happen
-function testConfiguration()
-    test_input = LinRange(-100f0, 100f0, 99)
-
-    try
-        for left in test_input
-            for right in test_input
-                for binop in binops
-                    test_output = binop.(left, right)
-                end
-            end
-            for unaop in unaops
-                test_output = unaop.(left)
-            end
-        end
-    catch error
-        @printf("\n\nYour configuration is invalid - one of your operators is not well-defined over the real line.\n\n\n")
-        throw(error)
-    end
-end
-
 
 function fullRun(niterations::Integer;
                 npop::Integer=300,

julia/truth.jl

@@ -0,0 +1,77 @@
+# *** Custom Functions
+##################################################################################################################################
+# *** Will somewhere need to define a list TRUTHS of all valid auxliary truths
+struct Transformation
+    type::Integer # 1 is symmetry, 2 is zero, 3 is equality
+    params::Array{Int32}
+    Transformation(type::Integer, params::Array{Int32}) = new(type, params)
+    Transformation(type::Integer, params::Array{Int64}) = new(type, params)
+
+end
+struct Truth
+    transformation::Transformation
+    weights::Array{Float32}
+    Truth(transformation::Transformation, weights::Array{Float32}) = new(transformation, weights)
+    Truth(type::Int64, params::Array{Int64}, weights::Array{Float32}) = new(Transformation(type, params), weights)
+    Truth(transformation::Transformation, weights::Array{Float64}) = new(transformation, weights)
+    Truth(type::Int64, params::Array{Int64}, weights::Array{Float64}) = new(Transformation(type, params), weights)
+end
+# Returns a linear combination when given X of shape nxd, y of shape nx1 is f(x) and w of shape d+2x1, result is shape nx1
+function LinearPrediction(cX::Array{Float32}, cy::Array{Float32}, w::Array{Float32})::Array{Float32}
+     preds = 0
+     for i in 1:ndims(cX)
+       preds = preds .+ cX[:,i].*w[i]
+       end
+     preds = preds .+ cy.*w[ndims(cX)+1]
+     return preds .+ w[ndims(cX)+2]
+end
+
+# Returns a copy of the data with the two specified columns swapped
+function swapColumns(cX::Array{Float32, 2}, a::Integer, b::Integer)::Array{Float32, 2}
+    X1 = copy(cX)
+    X1[:, a] = cX[:, b]
+    X1[:, b] = cX[:, a]
+    return X1
+end
+
+# Returns a copy of the data with the specified integers in the list set to value given
+function setVal(cX::Array{Float32, 2}, a::Array{Int32, 1}, val::Float32)::Array{Float32, 2}
+    X1 = copy(cX)
+    for i in 1:size(a)[1]
+        X1[:, a[i]] = fill!(cX[:, a[i]], val)
+    end
+    return X1
+end
+
+# Returns a copy of the data with the specified integer indices in the list set to the first item of that list
+function setEq(cX::Array{Float32, 2}, a::Array{Int32, 1})::Array{Float32, 2}
+    X1 = copy(cX)
+    val = X1[:, a[1]]
+    for i in 1:size(a)[1]
+        X1[:, a[i]] = val
+    end
+    return X1
+end
+
+# Takes in a dataset and returns the transformed version of it as per the specified type and parameters
+function transform(cX::Array{Float32, 2}, transformation::Transformation)::Array{Float32, 2}
+    if transformation.type==1 # then symmetry
+        a = transformation.params[1]
+        b = transformation.params[2]
+        return swapColumns(cX, a, b)
+    elseif transformation.type==2 # then zero condition
+        return setVal(cX, transformation.params, Float32(0))
+    elseif transformation.type == 3 # then equality condition
+        return setEq(cX, transformation.params)
+    else # Then error return X
+        return cX
+    end
+end
+function transform(cX::Array{Float32, 2}, truth::Truth)::Array{Float32, 2}
+    return transform(cX, truth.transformation)
+end
+
+# Takes in X that has been transformed and returns what the Truth projects the target values should be
+function truthPrediction(X_transformed::Array{Float32, 2}, cy::Array{Float32}, truth::Truth)::Array{Float32}
+    return LinearPrediction(X_transformed, cy, truth.weights)
+end

julia/truthPops.jl

@@ -0,0 +1,170 @@
+# Returns the MSE between the predictions and the truth provided targets for the given dataset
+function truthScore(member::PopMember, cX::Array{Float32, 2}, cy::Array{Float32}, truth::Truth)::Float32
+    transformed = transform(cX, truth)
+    targets = truthPrediction(transformed, cy, truth)
+    preds = evalTreeArray(member.tree, transformed)
+    return MSE(preds, targets)
+end
+
+# Assumes a dataset X, y for a given truth
+function truthScore(member::PopMember, truth::Truth)::Float32
+    return truthScore(member, X, y, truth)
+end
+
+# Assumes a list of Truths TRUTHS is defined. Performs the truthScore function for each of them and returns the average
+function truthScore(member::PopMember, cX::Array{Float32, 2}, cy::Array{Float32})::Float32
+    s = 0
+    for truth in TRUTHS
+        s += (truthScore(member, cX, cy, truth))/size(TRUTHS)[1]
+    end
+    return s
+end
+
+# Assumes list of Truths TRUTHS and dataset X, y are defined
+function truthScore(member::PopMember)::Float32
+    return truthScore(member, X, y)
+end
+# Returns the MSE between the predictions and the truth provided targets for the given dataset
+function truthScore(tree::Node, cX::Array{Float32, 2}, cy::Array{Float32}, truth::Truth)::Float32
+    transformed = transform(cX, truth)
+    targets = truthPrediction(transformed, cy, truth)
+    preds = evalTreeArray(tree, transformed)
+    return MSE(preds, targets)
+end
+
+# Assumes a dataset X, y for a given truth
+function truthScore(tree::Node, truth::Truth)::Float32
+    return truthScore(tree, X, y, truth)
+end
+
+# Assumes a list of Truths TRUTHS is defined. Performs the truthScore function for each of them and returns the average
+function truthScore(tree::Node, cX::Array{Float32, 2}, cy::Array{Float32})::Float32
+    s = 0
+    for truth in TRUTHS
+        s += (truthScore(tree, cX, cy, truth))/size(TRUTHS)[1]
+    end
+    return s
+end
+
+# Assumes list of Truths TRUTHS and dataset X, y are defined
+function truthScore(tree::Node)::Float32
+    return truthScore(tree, X, y)
+end
+
+# Returns true iff Truth Score is below a given threshold i.e truth is satisfied
+function testTruth(member::PopMember, truth::Truth, threshold::Float32=Float32(1.0e-8))::Bool
+    truthError = truthScore(member, truth)
+    #print(stringTree(member.tree), "\n")
+    #print(truth, ": ")
+    #print(truthError, "\n")
+    if truthError > threshold
+        #print("Returns False \n ----\n")
+        return false
+    else
+        #print("Returns True \n ----\n")
+        return true
+    end
+end
+
+# Returns a list of violating functions from assumed list TRUTHS
+function violatingTruths(member::PopMember)::Array{Truth}
+    return violatingTruths(member.tree)
+end
+
+# Returns true iff Truth Score is below a given threshold i.e truth is satisfied
+function testTruth(tree::Node, truth::Truth, threshold::Float32=Float32(1.0e-3))::Bool
+    truthError = truthScore(tree, truth)
+    if truthError > threshold
+        return false
+    else
+        return true
+    end
+end
+
+# Returns a list of violating functions from assumed list TRUTHS
+function violatingTruths(tree::Node)::Array{Truth}
+    toReturn = []
+    #print("\n Checking Equation ", stringTree(tree), "\n")
+    for truth in TRUTHS
+        test_truth = testTruth(tree, truth)
+        #print("Truth: ", truth, ": " , test_truth, "\n-----\n")
+        if !test_truth
+            append!(toReturn, [truth])
+        end
+    end
+    return toReturn
+end
+
+function randomIndex(cX::Array{Float32, 2}, k::Integer=10)::Array{Int32, 1}
+    indxs = sample([Int32(i) for i in 1:size(cX)[1]], k)
+    return indxs
+end
+
+function randomIndex(leng::Integer, k::Integer=10)::Array{Int32, 1}
+    indxs = sample([Int32(i) for i in 1:leng], k)
+    return indxs
+end
+
+function extendedX(cX::Array{Float32, 2}, truth::Truth, indx::Array{Int32, 1})::Array{Float32, 2}
+    workingcX = copy(cX)
+    X_slice = workingcX[indx, :]
+    X_transformed = transform(X_slice, truth)
+    return X_transformed
+end
+function extendedX(truth::Truth, indx::Array{Int32, 1})::Union{Array{Float32, 2}, Nothing}
+    return extendedX(OGX, truth, indx)
+end
+function extendedX(cX::Array{Float32, 2}, violatedTruths::Array{Truth}, indx::Array{Int32, 1})::Union{Array{Float32, 2}, Nothing}
+    if length(violatedTruths) == 0
+        return nothing
+    end
+    workingX = extendedX(cX, violatedTruths[1], indx)
+    for truth in violatedTruths[2:length(violatedTruths)]
+        workingX = vcat(workingX, extendedX(cX, truth, indx))
+    end
+    return workingX
+end
+function extendedX(violatedTruths::Array{Truth}, indx::Array{Int32, 1})::Union{Array{Float32, 2}, Nothing}
+    return extendedX(OGX, violatedTruths, indx)
+end
+function extendedX(tree::Node, indx::Array{Int32, 1})::Union{Array{Float32, 2}, Nothing}
+    violatedTruths = violatingTruths(tree)
+    return extendedX(violatedTruths, indx)
+end
+function extendedX(member::PopMember, indx::Array{Int32, 1})::Union{Array{Float32, 2}, Nothing}
+    return extendedX(member.tree, indx)
+end
+
+
+function extendedy(cX::Array{Float32, 2}, cy::Array{Float32}, truth::Truth, indx::Array{Int32, 1})::Union{Array{Float32}, Nothing}
+    cy = copy(cy)
+    cX = copy(cX)
+    X_slice = cX[indx, :]
+    y_slice = cy[indx]
+    X_transformed = transform(X_slice, truth)
+    y_transformed = truthPrediction(X_transformed, y_slice, truth)
+    return y_transformed
+end
+function extendedy(truth::Truth, indx::Array{Int32, 1})::Union{Array{Float32}, Nothing}
+    return extendedy(OGX, OGy, truth, indx)
+end
+function extendedy(cX::Array{Float32, 2}, cy::Array{Float32}, violatedTruths::Array{Truth}, indx::Array{Int32, 1})::Union{Array{Float32}, Nothing}
+    if length(violatedTruths) == 0
+        return nothing
+    end
+    workingy = extendedy(cX, cy, violatedTruths[1], indx)
+    for truth in violatedTruths[2:length(violatedTruths)]
+        workingy = vcat(workingy, extendedy(cX, cy, truth, indx))
+    end
+    return workingy
+end
+function extendedy(violatedTruths::Array{Truth}, indx::Array{Int32, 1})::Union{Array{Float32}, Nothing}
+    return extendedy(OGX,OGy, violatedTruths, indx)
+end
+function extendedy(tree::Node, indx::Array{Int32, 1})::Union{Array{Float32}, Nothing}
+    violatedTruths = violatingTruths(tree)
+    return extendedy(violatedTruths, indx)
+end
+function extendedy(member::PopMember, indx::Array{Int32, 1})::Union{Array{Float32}, Nothing}
+    return extendedy(member.tree, indx)
+end

pysr/sr.py

@@ -192,15 +192,7 @@ def pysr(X=None, y=None, weights=None,
         (as strings).
 
     """
-    if threads is not None:
-        raise ValueError("The threads kwarg is deprecated. Use procs.")
-    if limitPowComplexity:
-        raise ValueError("The limitPowComplexity kwarg is deprecated. Use constraints.")
-    if maxdepth is None:
-        maxdepth = maxsize
-    if equation_file is None:
-        date_time = datetime.now().strftime("%Y-%m-%d_%H%M%S.%f")[:-3]
-        equation_file = 'hall_of_fame_' + date_time + '.csv'
+    _raise_depreciation_errors(limitPowComplexity, threads)
 
     if isinstance(X, pd.DataFrame):
         variable_names = list(X.columns)
@@ -211,119 +203,165 @@ def pysr(X=None, y=None, weights=None,
     if len(X.shape) == 1:
         X = X[:, None]
 
-    # Check for potential errors before they happen
-    assert len(unary_operators) + len(binary_operators) > 0
-    assert len(X.shape) == 2
-    assert len(y.shape) == 1
-    assert X.shape[0] == y.shape[0]
-    if weights is not None:
-        assert len(weights.shape) == 1
-        assert X.shape[0] == weights.shape[0]
-    if use_custom_variable_names:
-        assert len(variable_names) == X.shape[1]
+    _check_assertions(X, binary_operators, unary_operators,
+                     use_custom_variable_names, variable_names, weights, y)
 
 
     if len(X) > 10000 and not batching:
         warnings.warn("Note: you are running with more than 10,000 datapoints. You should consider turning on batching (https://pysr.readthedocs.io/en/latest/docs/options/#batching). You should also reconsider if you need that many datapoints. Unless you have a large amount of noise (in which case you should smooth your dataset first), generally < 10,000 datapoints is enough to find a functional form with symbolic regression. More datapoints will lower the search speed.")
 
-    if select_k_features is not None:
-        selection = run_feature_selection(X, y, select_k_features)
-        print(f"Using features {selection}")
-        X = X[:, selection]
-
-        if use_custom_variable_names:
-            variable_names = [variable_names[selection[i]] for i in range(len(selection))]
+    X, variable_names = _handle_feature_selection(
+            X, select_k_features,
+            use_custom_variable_names, variable_names, y
+        )
 
+    if maxdepth is None:
+        maxdepth = maxsize
+    if equation_file is None:
+        date_time = datetime.now().strftime("%Y-%m-%d_%H%M%S.%f")[:-3]
+        equation_file = 'hall_of_fame_' + date_time + '.csv'
     if populations is None:
         populations = procs
+    if isinstance(binary_operators, str):
+        binary_operators = [binary_operators]
+    if isinstance(unary_operators, str):
+        unary_operators = [unary_operators]
+    if X is None:
+        X, y = _using_test_input(X, test, y)
+
+    kwargs = dict(X=X, y=y, weights=weights,
+                alpha=alpha, annealing=annealing, batchSize=batchSize,
+                 batching=batching, binary_operators=binary_operators,
+                 equation_file=equation_file, fast_cycle=fast_cycle,
+                 fractionReplaced=fractionReplaced,
+                 ncyclesperiteration=ncyclesperiteration,
+                 niterations=niterations, npop=npop,
+                 topn=topn, verbosity=verbosity,
+                 julia_optimization=julia_optimization, timeout=timeout,
+                 fractionReplacedHof=fractionReplacedHof,
+                 hofMigration=hofMigration,
+                 limitPowComplexity=limitPowComplexity, maxdepth=maxdepth,
+                 maxsize=maxsize, migration=migration, nrestarts=nrestarts,
+                 parsimony=parsimony, perturbationFactor=perturbationFactor,
+                 populations=populations, procs=procs,
+                 shouldOptimizeConstants=shouldOptimizeConstants,
+                 unary_operators=unary_operators, useFrequency=useFrequency,
+                 use_custom_variable_names=use_custom_variable_names,
+                 variable_names=variable_names, warmupMaxsize=warmupMaxsize,
+                 weightAddNode=weightAddNode,
+                 weightDeleteNode=weightDeleteNode,
+                 weightDoNothing=weightDoNothing,
+                 weightInsertNode=weightInsertNode,
+                 weightMutateConstant=weightMutateConstant,
+                 weightMutateOperator=weightMutateOperator,
+                 weightRandomize=weightRandomize,
+                 weightSimplify=weightSimplify,
+                 constraints=constraints,
+                 extra_sympy_mappings=extra_sympy_mappings)
+
+    kwargs = {**_set_paths(tempdir), **kwargs}
+
+    kwargs['def_hyperparams'] = _metaprogram_fast_operator(**kwargs)
+
+    _handle_constraints(**kwargs)
+
+    kwargs['constraints_str'] = _make_constraints_str(**kwargs)
+    kwargs['def_hyperparams'] = _make_hyperparams_julia_str(**kwargs)
+    kwargs['def_auxiliary'] = _make_auxiliary_julia_str(**kwargs)
+    kwargs['def_datasets'] = _make_datasets_julia_str(**kwargs)
+
+    _create_julia_files(**kwargs)
+    _final_pysr_process(**kwargs)
+    _set_globals(**kwargs)
 
-    if isinstance(binary_operators, str): binary_operators = [binary_operators]
-    if isinstance(unary_operators, str): unary_operators = [unary_operators]
+    if delete_tempfiles:
+        shutil.rmtree(kwargs['tmpdir'])
 
-    if X is None:
-        if test == 'simple1':
-            eval_str = "np.sign(X[:, 2])*np.abs(X[:, 2])**2.5 + 5*np.cos(X[:, 3]) - 5"
-        elif test == 'simple2':
-            eval_str = "np.sign(X[:, 2])*np.abs(X[:, 2])**3.5 + 1/(np.abs(X[:, 0])+1)"
-        elif test == 'simple3':
-            eval_str = "np.exp(X[:, 0]/2) + 12.0 + np.log(np.abs(X[:, 0])*10 + 1)"
-        elif test == 'simple4':
-            eval_str = "1.0 + 3*X[:, 0]**2 - 0.5*X[:, 0]**3 + 0.1*X[:, 0]**4"
-        elif test == 'simple5':
-            eval_str = "(np.exp(X[:, 3]) + 3)/(np.abs(X[:, 1]) + np.cos(X[:, 0]) + 1.1)"
-
-        X = np.random.randn(100, 5)*3
-        y = eval(eval_str)
-        print("Running on", eval_str)
+    return get_hof(**kwargs)
 
-    # System-independent paths
-    pkg_directory = Path(__file__).parents[1] / 'julia'
-    pkg_filename = pkg_directory / "sr.jl"
-    operator_filename = pkg_directory / "operators.jl"
 
-    tmpdir = Path(tempfile.mkdtemp(dir=tempdir))
-    hyperparam_filename = tmpdir / f'hyperparams.jl'
-    dataset_filename = tmpdir / f'dataset.jl'
-    runfile_filename = tmpdir / f'runfile.jl'
-    X_filename = tmpdir / "X.csv"
-    y_filename = tmpdir / "y.csv"
-    weights_filename = tmpdir / "weights.csv"
+def _make_auxiliary_julia_str(julia_auxiliary_filenames, **kwargs):
+    def_auxiliary = '\n'.join([
+        f"""include("{_escape_filename(aux_fname)}")""" for aux_fname in julia_auxiliary_filenames
+    ])
+    return def_auxiliary
 
-    def_hyperparams = ""
 
-    # Add pre-defined functions to Julia
-    for op_list in [binary_operators, unary_operators]:
-        for i in range(len(op_list)):
-            op = op_list[i]
-            is_user_defined_operator = '(' in op
+def _set_globals(X, equation_file, extra_sympy_mappings, variable_names, **kwargs):
+    global global_n_features
+    global global_equation_file
+    global global_variable_names
+    global global_extra_sympy_mappings
+    global_n_features = X.shape[1]
+    global_equation_file = equation_file
+    global_variable_names = variable_names
+    global_extra_sympy_mappings = extra_sympy_mappings
 
-            if is_user_defined_operator:
-                def_hyperparams += op + "\n"
-                # Cut off from the first non-alphanumeric char:
-                first_non_char = [
-                        j for j in range(len(op))
-                        if not (op[j].isalpha() or op[j].isdigit())][0]
-                function_name = op[:first_non_char]
-                op_list[i] = function_name
 
-    #arbitrary complexity by default
-    for op in unary_operators:
-        if op not in constraints:
-            constraints[op] = -1
-    for op in binary_operators:
-        if op not in constraints:
-            constraints[op] = (-1, -1)
-        if op in ['plus', 'sub']:
-            if constraints[op][0] != constraints[op][1]:
-                raise NotImplementedError("You need equal constraints on both sides for - and *, due to simplification strategies.")
-        elif op == 'mult':
-            # Make sure the complex expression is in the left side.
-            if constraints[op][0] == -1:
-                continue
-            elif constraints[op][1] == -1 or constraints[op][0] < constraints[op][1]:
-                constraints[op][0], constraints[op][1] = constraints[op][1], constraints[op][0]
+def _final_pysr_process(julia_optimization, procs, runfile_filename, timeout, **kwargs):
+    command = [
+        f'julia', f'-O{julia_optimization:d}',
+        f'-p', f'{procs}',
+        str(runfile_filename),
+    ]
+    if timeout is not None:
+        command = [f'timeout', f'{timeout}'] + command
+    print("Running on", ' '.join(command))
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=1)
+    try:
+        while True:
+            line = process.stdout.readline()
+            if not line: break
+            print(line.decode('utf-8').replace('\n', ''))
 
-    constraints_str = "const una_constraints = ["
-    first = True
-    for op in unary_operators:
-        val = constraints[op]
-        if not first:
-            constraints_str += ", "
-        constraints_str += f"{val:d}"
-        first = False
+        process.stdout.close()
+        process.wait()
+    except KeyboardInterrupt:
+        print("Killing process... will return when done.")
+        process.kill()
 
-    constraints_str += """]
-const bin_constraints = ["""
 
-    first = True
-    for op in binary_operators:
-        tup = constraints[op]
-        if not first:
-            constraints_str += ", "
-        constraints_str += f"({tup[0]:d}, {tup[1]:d})"
-        first = False
-    constraints_str += "]"
+def _create_julia_files(auxiliary_filename, dataset_filename, def_auxiliary, def_datasets, def_hyperparams, fractionReplaced, hyperparam_filename,
+                       ncyclesperiteration, niterations, npop, pkg_filename, runfile_filename, topn, verbosity, **kwargs):
+    with open(hyperparam_filename, 'w') as f:
+        print(def_hyperparams, file=f)
+    with open(dataset_filename, 'w') as f:
+        print(def_datasets, file=f)
+    with open(auxiliary_filename, 'w') as f:
+        print(def_auxiliary, file=f)
+    with open(runfile_filename, 'w') as f:
+        print(f'@everywhere include("{_escape_filename(hyperparam_filename)}")', file=f)
+        print(f'@everywhere include("{_escape_filename(dataset_filename)}")', file=f)
+        print(f'@everywhere include("{_escape_filename(auxiliary_filename)}")', file=f)
+        print(f'@everywhere include("{_escape_filename(pkg_filename)}")', file=f)
+        print(
+            f'fullRun({niterations:d}, npop={npop:d}, ncyclesperiteration={ncyclesperiteration:d}, fractionReplaced={fractionReplaced:f}f0, verbosity=round(Int32, {verbosity:f}), topn={topn:d})',
+            file=f)
+        print(f'rmprocs(nprocs)', file=f)
+
+
+def _make_datasets_julia_str(X, X_filename, weights, weights_filename, y, y_filename, **kwargs):
+    def_datasets = """using DelimitedFiles"""
+    np.savetxt(X_filename, X, delimiter=',')
+    np.savetxt(y_filename, y, delimiter=',')
+    if weights is not None:
+        np.savetxt(weights_filename, weights, delimiter=',')
+    def_datasets += f"""
+const X = readdlm("{_escape_filename(X_filename)}", ',', Float32, '\\n')
+const y = readdlm("{_escape_filename(y_filename)}", ',', Float32, '\\n')"""
+    if weights is not None:
+        def_datasets += f"""
+const weights = readdlm("{_escape_filename(weights_filename)}", ',', Float32, '\\n')"""
+    return def_datasets
 
+
+def _make_hyperparams_julia_str(X, alpha, annealing, batchSize, batching, binary_operators, constraints_str,
+                               def_hyperparams, equation_file, fast_cycle, fractionReplacedHof, hofMigration,
+                               limitPowComplexity, maxdepth, maxsize, migration, nrestarts, operator_filename,
+                               parsimony, perturbationFactor, populations, procs, shouldOptimizeConstants,
+                               unary_operators, useFrequency, use_custom_variable_names, variable_names, warmupMaxsize, weightAddNode,
+                               weightDeleteNode, weightDoNothing, weightInsertNode, weightMutateConstant,
+                               weightMutateOperator, weightRandomize, weightSimplify, weights, **kwargs):
     def_hyperparams += f"""include("{_escape_filename(operator_filename)}")
 {constraints_str}
 const binops = {'[' + ', '.join(binary_operators) + ']'}
@@ -362,7 +400,6 @@ const warmupMaxsize = {warmupMaxsize:d}
 const limitPowComplexity = {"true" if limitPowComplexity else "false"}
 const useFrequency = {"true" if useFrequency else "false"}
 """
-
     op_runner = ""
     if len(binary_operators) > 0:
         op_runner += """
@@ -373,14 +410,13 @@ const useFrequency = {"true" if useFrequency else "false"}
         end"""
         for i in range(1, len(binary_operators)):
             op_runner += f"""
-    elseif i === {i+1}
+    elseif i === {i + 1}
         @inbounds @simd for j=1:clen
             x[j] = {binary_operators[i]}(x[j], y[j])
         end"""
         op_runner += """
     end
 end"""
-
     if len(unary_operators) > 0:
         op_runner += """
 @inline function UNAOP!(x::Array{Float32, 1}, i::Int, clen::Int)
@@ -390,85 +426,160 @@ end"""
         end"""
         for i in range(1, len(unary_operators)):
             op_runner += f"""
-    elseif i === {i+1}
+    elseif i === {i + 1}
         @inbounds @simd for j=1:clen
             x[j] = {unary_operators[i]}(x[j])
         end"""
         op_runner += """
     end
 end"""
-
     def_hyperparams += op_runner
+    if use_custom_variable_names:
+        def_hyperparams += f"""
+    const varMap = {'["' + '", "'.join(variable_names) + '"]'}"""
+    return def_hyperparams
 
-    def_datasets = """using DelimitedFiles"""
-
-    np.savetxt(X_filename, X, delimiter=',')
-    np.savetxt(y_filename, y, delimiter=',')
-    if weights is not None:
-        np.savetxt(weights_filename, weights, delimiter=',')
 
-    def_datasets += f"""
-const X = readdlm("{_escape_filename(X_filename)}", ',', Float32, '\\n')
-const y = readdlm("{_escape_filename(y_filename)}", ',', Float32, '\\n')"""
+def _make_constraints_str(binary_operators, constraints, unary_operators, **kwargs):
+    constraints_str = "const una_constraints = ["
+    first = True
+    for op in unary_operators:
+        val = constraints[op]
+        if not first:
+            constraints_str += ", "
+        constraints_str += f"{val:d}"
+        first = False
+    constraints_str += """]
+const bin_constraints = ["""
+    first = True
+    for op in binary_operators:
+        tup = constraints[op]
+        if not first:
+            constraints_str += ", "
+        constraints_str += f"({tup[0]:d}, {tup[1]:d})"
+        first = False
+    constraints_str += "]"
+    return constraints_str
 
-    if weights is not None:
-        def_datasets += f"""
-const weights = readdlm("{_escape_filename(weights_filename)}", ',', Float32, '\\n')"""
 
-    if use_custom_variable_names:
-        def_hyperparams += f"""
-const varMap = {'["' + '", "'.join(variable_names) + '"]'}"""
+def _handle_constraints(binary_operators, constraints, unary_operators, **kwargs):
+    for op in unary_operators:
+        if op not in constraints:
+            constraints[op] = -1
+    for op in binary_operators:
+        if op not in constraints:
+            constraints[op] = (-1, -1)
+        if op in ['plus', 'sub']:
+            if constraints[op][0] != constraints[op][1]:
+                raise NotImplementedError(
+                    "You need equal constraints on both sides for - and *, due to simplification strategies.")
+        elif op == 'mult':
+            # Make sure the complex expression is in the left side.
+            if constraints[op][0] == -1:
+                continue
+            elif constraints[op][1] == -1 or constraints[op][0] < constraints[op][1]:
+                constraints[op][0], constraints[op][1] = constraints[op][1], constraints[op][0]
 
-    with open(hyperparam_filename, 'w') as f:
-        print(def_hyperparams, file=f)
 
-    with open(dataset_filename, 'w') as f:
-        print(def_datasets, file=f)
+def _metaprogram_fast_operator(binary_operators, unary_operators, **kwargs):
+    def_hyperparams = ""
+    for op_list in [binary_operators, unary_operators]:
+        for i in range(len(op_list)):
+            op = op_list[i]
+            is_user_defined_operator = '(' in op
 
-    with open(runfile_filename, 'w') as f:
-        print(f'@everywhere include("{_escape_filename(hyperparam_filename)}")', file=f)
-        print(f'@everywhere include("{_escape_filename(dataset_filename)}")', file=f)
-        print(f'@everywhere include("{_escape_filename(pkg_filename)}")', file=f)
-        print(f'fullRun({niterations:d}, npop={npop:d}, ncyclesperiteration={ncyclesperiteration:d}, fractionReplaced={fractionReplaced:f}f0, verbosity=round(Int32, {verbosity:f}), topn={topn:d})', file=f)
-        print(f'rmprocs(nprocs)', file=f)
+            if is_user_defined_operator:
+                def_hyperparams += op + "\n"
+                # Cut off from the first non-alphanumeric char:
+                first_non_char = [
+                    j for j in range(len(op))
+                    if not (op[j].isalpha() or op[j].isdigit())][0]
+                function_name = op[:first_non_char]
+                op_list[i] = function_name
+    return def_hyperparams
+
+
+def _using_test_input(X, test, y):
+    if test == 'simple1':
+        eval_str = "np.sign(X[:, 2])*np.abs(X[:, 2])**2.5 + 5*np.cos(X[:, 3]) - 5"
+    elif test == 'simple2':
+        eval_str = "np.sign(X[:, 2])*np.abs(X[:, 2])**3.5 + 1/(np.abs(X[:, 0])+1)"
+    elif test == 'simple3':
+        eval_str = "np.exp(X[:, 0]/2) + 12.0 + np.log(np.abs(X[:, 0])*10 + 1)"
+    elif test == 'simple4':
+        eval_str = "1.0 + 3*X[:, 0]**2 - 0.5*X[:, 0]**3 + 0.1*X[:, 0]**4"
+    elif test == 'simple5':
+        eval_str = "(np.exp(X[:, 3]) + 3)/(np.abs(X[:, 1]) + np.cos(X[:, 0]) + 1.1)"
+    X = np.random.randn(100, 5) * 3
+    y = eval(eval_str)
+    print("Running on", eval_str)
+    return X, y
+
+
+def _handle_feature_selection(X, select_k_features, use_custom_variable_names, variable_names, y):
+    if select_k_features is not None:
+        selection = run_feature_selection(X, y, select_k_features)
+        print(f"Using features {selection}")
+        X = X[:, selection]
 
+        if use_custom_variable_names:
+            variable_names = [variable_names[selection[i]] for i in range(len(selection))]
+    return X, variable_names
 
-    command = [
-        f'julia', f'-O{julia_optimization:d}',
-        f'-p', f'{procs}',
-        str(runfile_filename),
-        ]
-    if timeout is not None:
-        command = [f'timeout', f'{timeout}'] + command
 
-    global global_n_features
-    global global_equation_file
-    global global_variable_names
-    global global_extra_sympy_mappings
+def _set_paths(tempdir):
+    # System-independent paths
+    pkg_directory = Path(__file__).parents[1] / 'julia'
+    pkg_filename = pkg_directory / "sr.jl"
+    operator_filename = pkg_directory / "Operators.jl"
+    julia_auxiliaries = [
+        "Equation.jl", "ProgramConstants.jl",
+        "LossFunctions.jl", "Utils.jl", "EvaluateEquation.jl",
+        "MutationFunctions.jl", "SimplifyEquation.jl", "PopMember.jl",
+        "HallOfFame.jl", "CheckConstraints.jl", "Mutate.jl",
+        "Population.jl", "RegularizedEvolution.jl", "SingleIteration.jl",
+        "ConstantOptimization.jl"
+    ]
+    julia_auxiliary_filenames = [
+        pkg_directory / fname
+        for fname in julia_auxiliaries
+    ]
 
-    global_n_features = X.shape[1]
-    global_equation_file = equation_file
-    global_variable_names = variable_names
-    global_extra_sympy_mappings = extra_sympy_mappings
+    tmpdir = Path(tempfile.mkdtemp(dir=tempdir))
+    hyperparam_filename = tmpdir / f'hyperparams.jl'
+    dataset_filename = tmpdir / f'dataset.jl'
+    auxiliary_filename = tmpdir / f'auxiliary.jl'
+    runfile_filename = tmpdir / f'runfile.jl'
+    X_filename = tmpdir / "X.csv"
+    y_filename = tmpdir / "y.csv"
+    weights_filename = tmpdir / "weights.csv"
+    return dict(auxiliary_filename=auxiliary_filename, X_filename=X_filename,
+            dataset_filename=dataset_filename,
+            hyperparam_filename=hyperparam_filename,
+            julia_auxiliary_filenames=julia_auxiliary_filenames,
+            operator_filename=operator_filename, pkg_filename=pkg_filename,
+            runfile_filename=runfile_filename, tmpdir=tmpdir,
+            weights_filename=weights_filename, y_filename=y_filename)
 
-    print("Running on", ' '.join(command))
-    process = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=1)
-    try:
-        while True:
-            line = process.stdout.readline()
-            if not line: break
-            print(line.decode('utf-8').replace('\n', ''))
 
-        process.stdout.close()
-        process.wait()
-    except KeyboardInterrupt:
-        print("Killing process... will return when done.")
-        process.kill()
+def _check_assertions(X, binary_operators, unary_operators, use_custom_variable_names, variable_names, weights, y):
+    # Check for potential errors before they happen
+    assert len(unary_operators) + len(binary_operators) > 0
+    assert len(X.shape) == 2
+    assert len(y.shape) == 1
+    assert X.shape[0] == y.shape[0]
+    if weights is not None:
+        assert len(weights.shape) == 1
+        assert X.shape[0] == weights.shape[0]
+    if use_custom_variable_names:
+        assert len(variable_names) == X.shape[1]
 
-    if delete_tempfiles:
-        shutil.rmtree(tmpdir)
 
-    return get_hof()
+def _raise_depreciation_errors(limitPowComplexity, threads):
+    if threads is not None:
+        raise ValueError("The threads kwarg is deprecated. Use procs.")
+    if limitPowComplexity:
+        raise ValueError("The limitPowComplexity kwarg is deprecated. Use constraints.")
 
 
 def run_feature_selection(X, y, select_k_features):
@@ -485,7 +596,7 @@ def run_feature_selection(X, y, select_k_features):
             max_features=select_k_features, prefit=True)
     return selector.get_support(indices=True)
 
-def get_hof(equation_file=None, n_features=None, variable_names=None, extra_sympy_mappings=None):
+def get_hof(equation_file=None, n_features=None, variable_names=None, extra_sympy_mappings=None, **kwargs):
     """Get the equations from a hall of fame file. If no arguments
     entered, the ones used previously from a call to PySR will be used."""