First commit

eureqa.jl

@@ -0,0 +1,440 @@
+using ProgressBars
+
+# Define allowed operators
+plus(x::Float64, y::Float64) = x+y
+mult(x::Float64, y::Float64) = x*y;
+
+# (Apparently using const for globals helps speed)
+const binops = [plus, mult]
+const unaops = [sin, cos, exp];
+
+const nvar = 5;
+const X = rand(100, nvar);
+
+# Here is the function we want to learn (x2^2 + cos(x3) + 5)
+const y = ((cx,)->cx^2).(X[:, 2]) + cos.(X[:, 3]) .+ 5.0;
+
+# How much to punish complexity
+const parsimony = 0.01
+# How much to scale temperature by (T between 0 and 1)
+const alpha = 10.0
+
+
+
+
+id = (x,) -> x
+const nuna = size(unaops)[1]
+const nbin = size(binops)[1]
+const nops = nuna + nbin
+
+# Define a serialization format for the symbolic equations:
+mutable struct Node
+    #Holds operators, variables, constants in a tree
+    degree::Int #0 for constant/variable, 1 for cos/sin, 2 for +/* etc.
+    val::Union{Float64, Int} #Either const value, or enumerates variable
+    constant::Bool #false if variable 
+    op::Function #enumerates operator (for degree=1,2)
+    l::Union{Node, Nothing}
+    r::Union{Node, Nothing}
+    
+    Node(val::Float64) = new(0, val, true, id, nothing, nothing)
+    Node(val::Int) = new(0, val, false, id, nothing, nothing)
+    Node(op, l::Node) = new(1, 0.0, false, op, l, nothing)
+    Node(op, l::Union{Float64, Int}) = new(1, 0.0, false, op, Node(l), nothing)
+    Node(op, l::Node, r::Node) = new(2, 0.0, false, op, l, r)
+    
+    #Allow to pass the leaf value without additional node call:
+    Node(op, l::Union{Float64, Int}, r::Node) = new(2, 0.0, false, op, Node(l), r)
+    Node(op, l::Node, r::Union{Float64, Int}) = new(2, 0.0, false, op, l, Node(r))
+    Node(op, l::Union{Float64, Int}, r::Union{Float64, Int}) = new(2, 0.0, false, op, Node(l), Node(r))
+end
+
+# Evaluate a symbolic equation:
+function evalTree(tree::Node, x::Array{Float64, 1}=Float64[])::Float64
+    if tree.degree == 0
+        if tree.constant
+            return tree.val
+        else
+            return x[tree.val]
+        end
+    elseif tree.degree == 1
+        return tree.op(evalTree(tree.l, x))
+    else
+        return tree.op(evalTree(tree.l, x), evalTree(tree.r, x))
+    end
+end
+
+# Count the operators, constants, variables in an equation
+function countNodes(tree::Node)::Int
+    if tree.degree == 0
+        return 1
+    elseif tree.degree == 1
+        return 1 + countNodes(tree.l)
+    else
+        return 1 + countNodes(tree.l) + countNodes(tree.r)
+    end
+end
+
+# Convert an equation to a string
+function stringTree(tree::Node)::String
+    if tree.degree == 0
+        if tree.constant
+            return string(tree.val)
+        else
+            return "x$(tree.val)"
+        end
+    elseif tree.degree == 1
+        return "$(tree.op)($(stringTree(tree.l)))"
+    else
+        return "$(tree.op)($(stringTree(tree.l)), $(stringTree(tree.r)))"
+    end
+end
+
+# Print an equation
+function printTree(tree::Node)
+    println(stringTree(tree))
+end
+
+# Return a random node from the tree
+function randomNode(tree::Node)::Node
+    if tree.degree == 0
+        return tree
+    end
+    a = countNodes(tree)
+    b = 0
+    c = 0
+    if tree.degree >= 1
+        b = countNodes(tree.l)
+    end
+    if tree.degree == 2
+        c = countNodes(tree.r)
+    end
+            
+    i = rand(1:1+b+c)
+    if i <= b
+        return randomNode(tree.l)
+    elseif i == b + 1
+        return tree
+    end
+    
+    return randomNode(tree.r)        
+end
+
+# Count the number of unary operators in the equation
+function countUnaryOperators(tree::Node)::Int
+    if tree.degree == 0
+        return 0
+    elseif tree.degree == 1
+        return 1 + countUnaryOperators(tree.l)
+    else
+        return 0 + countUnaryOperators(tree.l) + countUnaryOperators(tree.r)
+    end
+end
+
+# Count the number of binary operators in the equation
+function countBinaryOperators(tree::Node)::Int
+    if tree.degree == 0
+        return 0
+    elseif tree.degree == 1
+        return 0 + countBinaryOperators(tree.l)
+    else
+        return 1 + countBinaryOperators(tree.l) + countBinaryOperators(tree.r)
+    end
+end
+
+# Count the number of operators in the equation
+function countOperators(tree::Node)::Int
+    return countUnaryOperators(tree) + countBinaryOperators(tree)
+end
+
+# Randomly convert an operator into another one (binary->binary;
+# unary->unary)
+function mutateOperator(tree::Node)::Node
+    if countOperators(tree) == 0
+        return tree
+    end
+    node = randomNode(tree)
+    while node.degree == 0
+        node = randomNode(tree)
+    end
+    if node.degree == 1
+        node.op = unaops[rand(1:length(unaops))]
+    else
+        node.op = binops[rand(1:length(binops))]
+    end
+    return tree
+end
+
+# Count the number of constants in an equation
+function countConstants(tree::Node)::Int
+    if tree.degree == 0
+        return convert(Int, tree.constant)
+    elseif tree.degree == 1
+        return 0 + countConstants(tree.l)
+    else
+        return 0 + countConstants(tree.l) + countConstants(tree.r)
+    end
+end
+
+# Randomly perturb a constant
+function mutateConstant(
+        tree::Node, T::Float64,
+        probNegate::Float64=0.01)::Node
+    # T is between 0 and 1.
+    
+    if countConstants(tree) == 0
+        return tree
+    end
+    node = randomNode(tree)
+    while node.degree != 0 || node.constant == false
+        node = randomNode(tree)
+    end
+    
+    maxChange = T + 1.0
+    factor = maxChange^rand()
+    makeConstBigger = rand() > 0.5
+    
+    if makeConstBigger 
+        node.val *= factor
+    else
+        node.val /= factor
+    end
+    
+    if rand() > probNegate
+        node.val *= -1
+    end
+    
+    return tree
+end
+
+# Evaluate an equation over an array of datapoints
+function evalTreeArray(
+        tree::Node,
+        x::Array{Float64, 2})::Array{Float64, 1}
+    return mapslices(
+        (cx,) -> evalTree(tree, cx),
+        x,
+        dims=[2]
+    )[:, 1]
+end
+
+# Sum of square error between two arrays
+function SSE(x::Array{Float64}, y::Array{Float64})::Float64
+    return sum(((cx,)->cx^2).(x - y))
+end
+
+# Mean of square error between two arrays
+function MSE(x::Array{Float64}, y::Array{Float64})::Float64
+    return SSE(x, y)/size(x)[1]
+end
+
+# Score an equation
+function scoreFunc(
+        tree::Node,
+        X::Array{Float64, 2},
+        y::Array{Float64, 1},
+        parsimony::Float64=0.1)::Float64
+    return MSE(evalTreeArray(tree, X), y) + countNodes(tree)*parsimony
+end
+
+# Add a random unary/binary operation to the end of a tree
+function appendRandomOp(tree::Node)::Node
+    node = randomNode(tree)
+    while node.degree != 0
+        node = randomNode(tree)
+    end
+    
+    choice = rand()
+    makeNewBinOp = choice < nbin/nops
+    if rand() > 0.5
+        left = randn()
+    else
+        left = rand(1:nvar)
+    end
+    if rand() > 0.5
+        right = randn()
+    else
+        right = rand(1:nvar)
+    end
+    
+    if makeNewBinOp
+        newnode = Node(
+            binops[rand(1:length(binops))],
+            left,
+            right
+        )
+    else
+        newnode = Node(
+            unaops[rand(1:length(unaops))],
+            left
+        )
+    end
+    node.l = newnode.l
+    node.r = newnode.r
+    node.op = newnode.op
+    node.degree = newnode.degree
+    node.val = newnode.val
+    node.constant = newnode.constant
+    return tree
+end
+
+# Select a random node, and replace it an the subtree
+# with a variable or constant
+function deleteRandomOp(tree::Node)::Node
+    node = randomNode(tree)
+    # Can "delete" variable or constant too
+    if rand() > 0.5
+        val = randn()
+    else
+        val = rand(1:nvar)
+    end
+    newnode = Node(val)
+    node.l = newnode.l
+    node.r = newnode.r
+    node.op = newnode.op
+    node.degree = newnode.degree
+    node.val = newnode.val
+    node.constant = newnode.constant
+    return tree
+end
+
+# Go through one simulated annealing mutation cycle
+#  exp(-delta/T) defines probability of accepting a change
+function iterate(
+        tree::Node, T::Float64,
+        X::Array{Float64, 2}, y::Array{Float64, 1},
+        alpha::Float64=1.0,
+        mult::Float64=0.1
+    )::Node
+    prev = deepcopy(tree)
+    
+    mutationChoice = rand()
+    weights = [8, 1, 1, 1]
+    weights /= sum(weights)
+    cweights = cumsum(weights)
+    
+    if mutationChoice < cweights[1]
+        tree = mutateConstant(tree, T)
+    elseif mutationChoice < cweights[2]
+        tree = mutateOperator(tree)
+    elseif mutationChoice < cweights[3]
+        tree = appendRandomOp(tree)
+    elseif mutationChoice < cweights[4]
+        tree = deleteRandomOp(tree)
+    end
+    
+    try
+        beforeLoss = scoreFunc(prev, X, y, mult)
+        afterLoss = scoreFunc(tree, X, y, mult)
+        delta = afterLoss - beforeLoss
+        probChange = exp(-delta/(T*alpha))
+
+        if probChange > rand()
+            return tree
+        end
+
+        return prev
+    catch error
+        # Sometimes too many chained exp operators
+        if isa(error, DomainError)
+            return prev
+        else
+            throw(error)
+        end
+    end
+end
+
+# Create a random equation by appending random operators
+function genRandomTree(length::Int)::Node
+    tree = Node(1.0)
+    for i=1:length
+        tree = appendRandomOp(tree)
+    end
+    return tree
+end
+
+
+# Define a member of population by equation, score, and age
+mutable struct PopMember
+    tree::Node
+    score::Float64
+    birth::Float64
+    
+    PopMember(t) = new(t, scoreFunc(t, X, y, parsimony), time()-1.6e9)
+end
+
+# A list of members of the population, with easy constructors,
+#  which allow for random generation of new populations
+mutable struct Population
+    members::Array{PopMember, 1}
+    n::Int
+    
+    Population(pop::Array{PopMember, 1}) = new(pop, size(pop)[1])
+    Population(npop::Int64) = new([PopMember(genRandomTree(3)) for i=1:npop], npop)
+    Population(npop::Int64, nlength::Int64) = new([PopMember(genRandomTree(nlength)) for i=1:npop], npop)
+    
+end
+
+# Sample 10 random members of the population, and make a new one
+function samplePop(pop::Population)::Population
+    idx = rand(1:pop.n, 10)
+    return Population(pop.members[idx])#Population(deepcopy(pop.members[idx]))
+end
+
+# Sample the population, and get the best member from that sample
+function bestOfSample(pop::Population)::PopMember
+    sample = samplePop(pop)
+    best_idx = argmin([sample.members[member].score for member=1:sample.n])
+    return sample.members[best_idx]
+end
+
+# Mutate the best sampled member of the population
+function iterateSample(pop::Population, T::Float64)::PopMember
+    allstar = bestOfSample(pop)
+    new = iterate(allstar.tree, T, X, y, alpha, parsimony)
+    allstar.tree = new
+    allstar.score = scoreFunc(new, X, y, parsimony)
+    allstar.birth = time() - 1.6e9
+    return allstar
+end
+
+# Pass through the population several times, replacing the oldest
+# with the fittest of a small subsample
+function regEvolCycle(pop::Population, T::Float64)::Population
+    for i=1:Int(pop.n/10)
+        baby = iterateSample(pop, T)
+        oldest = argmin([pop.members[member].birth for member=1:pop.n])
+        pop.members[oldest] = baby
+    end
+    return pop
+end
+
+# Cycle through regularized evolution many times,
+# printing the fittest equation every 10% through
+function run(ncycles::Int,
+        npop::Int=100,
+        annealing::Bool=false)::Population
+
+    allT = LinRange(1.0, 0.0, ncycles)
+    pop = Population(npop, 3)
+    bestScore = Inf
+    for iT in tqdm(1:size(allT)[1])
+        if annealing
+            pop = regEvolCycle(pop, allT[iT])
+        else
+            pop = regEvolCycle(pop, 0.0)
+        end
+        bestCurScoreIdx = argmin([pop.members[member].score for member=1:pop.n])
+        bestCurScore = pop.members[bestCurScoreIdx].score
+        if bestCurScore < bestScore
+            bestScore = bestCurScore
+            println(bestScore, " is the score for ", stringTree(pop.members[bestCurScoreIdx].tree))
+        end
+    end
+    return pop
+end
+
+println("Lets try to learn (x2^2 + cos(x3) + 5) using regularized evolution from scratch")
+pop = run(10000, 1000, false);
+
+