New experiments features (model/agent/representation comparison, ...)

.gitignore

@@ -38,4 +38,23 @@ exp_*/
 *.png
 
 # Plot files
-*plot.ipynb
+*plot.ipynb
+*plot.py
+
+# Save experiment files
+*save_experiment.py
+
+# nohup output
+nohup*.out
+
+learning_cp/tsptw/old_exp/
+learning_cp/tsptw/TSPTW*
+# all run files
+run*.jl
+*.sh
+learning_cp/comparison/plots.jl
+
+# all .out files (nohup output)
+*.out
+
+learning_cp/comparison/tensorboard_logs

learning_cp/common/benchmark.jl

@@ -0,0 +1,142 @@
+# This util evaluates a trained model on n new instances with ILDS(1), ILDS(2), ILDS with given budget and DFS
+# Any heuristic using specific state representation should have "specific" in its file name
+#include("../common/experiment.jl")
+include("../comparison/comparison.jl")
+include("utils.jl")
+
+#include("experiment.jl")
+#include("../comparison/comparison.jl")
+#include("utils.jl")
+using CUDA
+
+# Parameters to edit
+folder = "/home/martom/SeaPearl/SeaPearlZoo/learning_cp/comparison/2022-08-17/exp_MaxCut_100_10_4->10_4_10012-02-09/"
+
+chosen_features = Dict(
+"node_number_of_neighbors" => true,
+"constraint_type" => true,
+"constraint_activity" => true,
+"nb_not_bounded_variable" => true,
+"variable_initial_domain_size" => true,
+"variable_domain_size" => true,
+"variable_is_objective" => true,
+"variable_assigned_value" => true,
+"variable_is_bound" => true,
+"values_raw" => true)
+
+n = 20
+k = 4
+generator = SeaPearl.MaxCutGenerator(n,k)
+n = 10 # Number of instances to evaluate on
+budget = 10000 # Budget of visited nodes
+has_objective = false # Set it to true if we have to branch on the objective variable
+include_dfs = false # Set it to true if you want to evaluate with DFS in addition to ILDS
+
+# Define your basic heuristics here
+threshold = 2*k
+MISHeuristic(x; cpmodel=nothing) = length(x.onDomainChange) - 1 < threshold ? 1 : 0
+heuristic_mis = SeaPearl.BasicHeuristic(MISHeuristic)
+selectMax(x::SeaPearl.IntVar; cpmodel=nothing) = SeaPearl.maximum(x.domain)
+heuristic_max = SeaPearl.BasicHeuristic(selectMax)
+basicHeuristics = OrderedDict(
+    "random" => SeaPearl.RandomHeuristic()
+    )
+
+function benchmark(folder::String, n::Int, chosen_features, has_objective::Bool, generator, basicHeuristics, include_dfs, budget::Int; verbose=true, ILDS = nothing)
+    models=[]
+    models_names=[]
+    for file in readdir(folder)
+        if splitext(file)[2] == ".bson"
+            @load folder * "/" * file model
+            push!(models, model)
+            push!(models_names, replace(splitext(file)[1], "model_"=>""))
+        end
+    end
+
+    reward = SeaPearl.GeneralReward
+
+    eval_strategies = SeaPearl.SearchStrategy[]
+    search_strategy_names = String[]
+    if typeof(ILDS) == SeaPearl.ILDSearch
+        for i in 0:ILDS.d
+            push!(eval_strategies, SeaPearl.ILDSearch(i))
+            push!(search_strategy_names, "ILDS"*string(i))
+        end 
+        push!(eval_strategies, SeaPearl.ILDSearch(10))
+        append!(search_strategy_names, ["ILDSbudget", "DFS"])
+    else
+        eval_strategies = SeaPearl.SearchStrategy[SeaPearl.ILDSearch(0),SeaPearl.ILDSearch(1),SeaPearl.ILDSearch(2),SeaPearl.ILDSearch(10)]
+        search_strategy_names = ["ILDS0", "ILDS1", "ILDS2", "ILDSbudget", "DFS"]
+        if include_dfs
+            push!(eval_strategies,SeaPearl.DFSearch())
+        end
+    end 
+
+    agents = []
+    valueSelectionArray = SeaPearl.ValueSelection[]
+
+    for (i,model) in enumerate(models)
+        agent = RL.Agent(
+            policy=RL.QBasedPolicy(
+                learner=RL.DQNLearner(
+                    approximator=RL.NeuralNetworkApproximator(
+                        model=model,
+                        optimizer=ADAM()
+                    ) |> cpu,
+                    target_approximator=RL.NeuralNetworkApproximator(
+                        model=model,
+                        optimizer=ADAM()
+                    ) |> cpu,
+                    loss_func=Flux.Losses.huber_loss
+                ),
+                explorer=RL.EpsilonGreedyExplorer(
+                    ϵ_stable=0.0
+                )
+            ),
+            trajectory=RL.CircularArraySLARTTrajectory(
+                capacity=1,
+                state=SeaPearl.DefaultTrajectoryState[] => (),
+                legal_actions_mask=Vector{Bool} => (1,),
+            )
+            )
+        push!(agents,agent)
+        if occursin("specific",models_names[i])
+            if isa(generator, SeaPearl.MaximumIndependentSetGenerator) 
+                state_representation = SeaPearl.MISStateRepresentation{SeaPearl.MISFeaturization,SeaPearl.DefaultTrajectoryState}
+            elseif isa(generator, SeaPearl.LegacyGraphColoringGenerator) || isa(generator, SeaPearl.HomogeneousGraphColoringGenerator) || isa(generator, SeaPearl.ClusterizedGraphColoringGenerator) || isa(generator, SeaPearl.BarabasiAlbertGraphGenerator) || isa(generator, SeaPearl.ErdosRenyiGraphGenerator) || isa(generator, SeaPearl.WattsStrogatzGraphGenerator)
+                state_representation = SeaPearl.GraphColoringStateRepresentation{SeaPearl.GraphColoringFeaturization, SeaPearl.DefaultTrajectoryState}
+            end
+            push!(valueSelectionArray, SeaPearl.SimpleLearnedHeuristic{state_representation,reward,SeaPearl.FixedOutput}(agent))
+        else
+            state_representation = SeaPearl.HeterogeneousStateRepresentation{SeaPearl.DefaultFeaturization, SeaPearl.HeterogeneousTrajectoryState}
+            push!(valueSelectionArray, SeaPearl.SimpleLearnedHeuristic{state_representation,reward,SeaPearl.FixedOutput}(agent; chosen_features=chosen_features))
+        end
+    end
+    append!(valueSelectionArray, collect(values(basicHeuristics)))
+    append!(models_names, collect(keys(basicHeuristics)))
+    variableHeuristic = SeaPearl.MinDomainVariableSelection{has_objective}()
+    evaluator = SeaPearl.SameInstancesEvaluator(valueSelectionArray, generator; nbInstances=n)
+    folder_names = split(folder, "/")
+    #println(pwd())
+    #println(folder)
+    if !isdir("../benchmarks/"*folder_names[1])
+        mkdir("../benchmarks/"*folder_names[1])
+    end
+    dir = mkdir("../benchmarks/"*folder)
+    for (j, search_strategy) in enumerate(eval_strategies)
+        if search_strategy == SeaPearl.ILDSearch(10)
+            SeaPearl.setNodesBudget!(evaluator, budget)
+        end
+        SeaPearl.evaluate(evaluator, variableHeuristic, search_strategy; verbose = verbose)
+        eval_metrics = evaluator.metrics
+        for i in 1:size(eval_metrics)[2]
+            SeaPearlExtras.storedata(eval_metrics[:, i]; filename= dir *"/"* search_strategy_names[j] * "_" * models_names[i])
+        end
+        # empty evaluator metrics
+        if search_strategy == SeaPearl.ILDSearch(10)
+            SeaPearl.resetNodesBudget!(evaluator)
+        end
+        empty!(evaluator)
+    end
+end
+#benchmark(folder, n, chosen_features, has_objective, generator, basicHeuristics, include_dfs, budget)

learning_cp/common/experiment.jl

@@ -0,0 +1,147 @@
+using SeaPearl
+using SeaPearlExtras
+using ReinforcementLearning
+const RL = ReinforcementLearning
+using Flux
+using GeometricFlux
+using JSON
+using BSON: @save, @load
+using Dates
+using Random
+using LightGraphs
+using OrderedCollections
+using CircularArrayBuffers
+using PyCall
+using Pkg
+using TensorBoardLogger, Logging
+
+"""
+ENV["PYTHON"] = "/home/x86_64-unknown-linux_ol7-gnu/anaconda-2022.05/bin/python"
+using Pkg
+Pkg.build("PyCall")
+run(`$(PyCall.python) -m pip install matplotlib`)
+run(`$(PyCall.python) -m pip install pandas`)
+run(`$(PyCall.python) -m pip install seaborn`)
+run(`$(PyCall.python) -m pip install ipython`)
+"""
+
+# -------------------
+# -------------------
+# Core function
+# -------------------
+# -------------------
+
+function trytrain(; nbEpisodes::Int, evalFreq::Int, nbInstances::Int, restartPerInstances::Int=1, generator::SeaPearl.AbstractModelGenerator, variableHeuristic::SeaPearl.AbstractVariableSelection=SeaPearl.MinDomainVariableSelection{false}(), learnedHeuristics::OrderedDict{String,<:SeaPearl.LearnedHeuristic}, basicHeuristics::OrderedDict{String,SeaPearl.BasicHeuristic}, base_name="experiment"::String, exp_name=""::String, out_solver=true::Bool, verbose=false::Bool, nbRandomHeuristics=0::Int, eval_timeout=nothing::Union{Nothing, Int},  training_timeout=nothing::Union{Nothing, Int}, eval_every =nothing::Union{Nothing, Int}, eval_strategy=SeaPearl.DFSearch(), strategy = SeaPearl.DFSearch(), seedTraining = nothing::Union{Nothing, Int}, seedEval =  nothing, eval_generator=nothing, logger = nothing, device = cpu)
+
+
+    experienceTime = Base.replace("$(round(now(), Dates.Second(3)))", ":" => "-")
+    date = split(experienceTime, "T")[1]
+    time = split(experienceTime, "T")[2]
+    logger =TBLogger("tensorboard_logs/"*exp_name*date*"_"*time, min_level=Logging.Info)
+
+    if !isdir(date)
+        mkdir(date)
+    end
+    dir = mkdir(string(date, "/exp_", exp_name, time))
+    lh = last(collect(values(learnedHeuristics)))
+    code_dir = mkdir(dir*"/code/")
+    for file in readdir(".")
+        if isfile(file)
+            cp(file, code_dir*file)
+        end
+    end
+
+    randomHeuristics = Array{SeaPearl.BasicHeuristic}(undef, 0)
+    for i in 1:nbRandomHeuristics
+        push!(randomHeuristics, SeaPearl.RandomHeuristic())
+    end
+
+    valueSelectionArray = cat(collect(values(learnedHeuristics)), collect(values(basicHeuristics)), randomHeuristics, dims=1)
+
+    if !isnothing(eval_generator)
+        evaluator = SeaPearl.SameInstancesEvaluator(valueSelectionArray, eval_generator; evalFreq=evalFreq, nbInstances=nbInstances, evalTimeOut = eval_timeout, rng = MersenneTwister(seedEval) )
+    else
+        evaluator = SeaPearl.SameInstancesEvaluator(valueSelectionArray, generator; evalFreq=evalFreq, nbInstances=nbInstances, evalTimeOut = eval_timeout, rng = MersenneTwister(seedEval))
+    end
+    metricsArray, eval_metricsArray = SeaPearl.train!(
+        valueSelectionArray=valueSelectionArray,
+        generator=generator,
+        nbEpisodes=nbEpisodes,
+        strategy=strategy,
+        eval_strategy = eval_strategy,
+        variableHeuristic=variableHeuristic,
+        out_solver=out_solver,
+        verbose=verbose,
+        evaluator = evaluator,
+        training_timeout = training_timeout,
+        restartPerInstances=restartPerInstances,
+        rngTraining = MersenneTwister(seedTraining), 
+        eval_every = eval_every, 
+        logger = logger,
+        device = device,
+    )
+
+    #saving model weights
+    for (key, lh) in learnedHeuristics
+        if (hasfield(typeof(lh.agent.policy),:approximator)) #PPO
+            model = Flux.cpu(lh.agent.policy.approximator)
+        else #DQN
+            model = Flux.cpu(lh.agent.policy.learner.approximator)
+        end
+        @save dir * "/model_" * key * ".bson" model
+    end
+
+    counter = 0
+    for key in keys(learnedHeuristics)
+        counter += 1
+        SeaPearlExtras.storedata(metricsArray[counter]; filename=dir * "/" * base_name * "_training_" * key)
+    end
+
+    counter = 0
+    for key in keys(learnedHeuristics)
+        counter += 1
+        SeaPearlExtras.storedata(eval_metricsArray[:, counter]; filename=dir * "/" * base_name * "_" * key)
+    end
+    for key in keys(basicHeuristics)
+        counter += 1
+        SeaPearlExtras.storedata(eval_metricsArray[:, counter]; filename=dir * "/" * base_name * "_" * key)
+    end
+    for i = 1:nbRandomHeuristics
+        SeaPearlExtras.storedata(eval_metricsArray[:, counter+i]; filename=dir * "/" * base_name * "_random$(i)")
+    end
+
+    py"""
+    import sys
+    sys.path.insert(0, "/home/martom/SeaPearl/SeaPearlExtras.jl/src/metrics/basicmetrics/")
+    from benchmarkPy import *
+    import plots
+    import numpy as np
+    def benchmark(path):
+        print_all(path +"/")
+
+    def plot(path,eval):
+        plots.all(path +"/", window=100, estimator=np.mean, ilds= eval)
+    """
+
+    py"plot"(dir, eval_strategy != SeaPearl.DFSearch())
+
+    chosen_features = valueSelectionArray[1].chosen_features
+    feature_size = [6, 5, 2]
+
+    n = 10 # Number of instances to evaluate on
+    budget = 1000 # Budget of visited nodes
+    has_objective = false # Set it to true if we have to branch on the objective variable
+    include_dfs = (eval_strategy == SeaPearl.DFSearch()) # Set it to true if you want to evaluate with DFS in addition to ILDS
+
+    selectMin(x::SeaPearl.IntVar; cpmodel=nothing) = SeaPearl.minimum(x.domain)
+    heuristic_min = SeaPearl.BasicHeuristic(selectMin)
+    basicHeuristics = OrderedDict(
+        "random" => SeaPearl.RandomHeuristic()
+        )
+    include("../common/benchmark.jl")
+    Base.invokelatest(benchmark, dir, n, chosen_features, has_objective, generator, basicHeuristics, include_dfs, budget; ILDS = eval_strategy)
+
+    py"benchmark"(dir)
+
+    return metricsArray, eval_metricsArray
+end