Add extra confs

Author: SCiarella

Project.toml

@@ -31,6 +31,8 @@ NeuralClosure = "099dac27-d7f2-4047-93d5-0baee36b9c25"
 Observables = "510215fc-4207-5dde-b226-833fc4488ee2"
 OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95"
 Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
+OptimizationCMAEvolutionStrategy = "bd407f91-200f-4536-9381-e4ba712f53f8"
+OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e"
 ParameterSchedulers = "d7d3b36b-41b8-4d0d-a2bf-768c6151755e"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -73,6 +75,8 @@ NeuralClosure = "1.0.0"
 Observables = "0.5"
 OpenSSL_jll = "3.0.13"
 Optimisers = "0.4"
+OptimizationCMAEvolutionStrategy = "0.3.0"
+OptimizationOptimJL = "0.4.3"
 ParameterSchedulers = "0.4"
 Statistics = "1.11.1"
 cuDNN = "1"

cnn_model_workflow.jl

@@ -101,6 +101,9 @@ using Lux
 using LuxCUDA
 using NNlib
 using Optimisers
+using Optimisers: Adam
+using OptimizationOptimJL
+using OptimizationCMAEvolutionStrategy
 using ParameterSchedulers
 using Random
 
@@ -248,6 +251,11 @@ if haskey(conf["priori"], "reuse")
     @info "Reuse a-priori training from closure named: $reuse"
     reusepriorfile(reuse, outdir, closure_name)
 end
+if haskey(conf["posteriori"], "reuse")
+    reuse = conf["posteriori"]["reuse"]
+    @info "Reuse a-posteriori training from closure named: $reuse"
+    reusepostfile(reuse, outdir, closure_name)
+end
 
 # Train
 for i = 1:ntrajectory

configs/snellius/cnn_CMA.yaml

@@ -0,0 +1,55 @@
+docreatedata: true
+docomp: true
+ntrajectory: 8
+T: "Float32"
+dataproj: true
+params:
+  D: 2
+  lims: [0.0, 1.0]
+  Re: 6000.0
+  tburn: 0.5
+  tsim: 5.0
+  savefreq: 50
+  ndns: 4096
+  nles: [64]
+  filters: ["FaceAverage()"]
+  icfunc: "(setup, psolver, rng) -> random_field(setup, T(0); kp=20, psolver, rng)"
+  method: "RKMethods.Wray3(; T)"
+  bodyforce: "(dim, x, y, t) -> (dim == 1) * 5 * sinpi(8 * y)"
+  issteadybodyforce: true
+  processors: "(; log = timelogger(; nupdate=100))"
+  Δt: 0.00005
+seeds:
+  dns: 123456
+  θ_start: 234
+  prior: 345
+  post: 456
+closure:
+  name: "cnn_cma"
+  type: cnn
+  radii: [2, 2, 2, 2, 2]
+  channels: [24, 24, 24, 24, 2]
+  activations: ["tanh", "tanh", "tanh", "tanh", "identity"]
+  use_bias: [true, true, true, true, false]
+  rng: "Xoshiro(seeds.θ_start)"
+priori:
+  reuse: "cnn_project"
+  dotrain: true
+  nepoch: 50000
+  batchsize: 64
+  opt: "OptimiserChain(Adam(T(1.0e-3)), ClipGrad(0.1))"
+  do_plot: false
+  plot_train: false
+posteriori:
+  reuse: "cnn_project"
+  dotrain: true
+  projectorders: "(ProjectOrder.Last, )"
+  nepoch: 2000
+  opt: "OptimizationCMAEvolutionStrategy.CMAEvolutionStrategyOpt()"
+  nunroll: 5
+  nunroll_valid: 10
+  nsamples: 1
+  dt: 0.0001
+  do_plot: false
+  plot_train: false
+  sensealg: "InterpolatingAdjoint()"

configs/snellius/cnn_backsol.yaml

@@ -0,0 +1,55 @@
+docreatedata: true
+docomp: true
+ntrajectory: 8
+T: "Float32"
+dataproj: true
+params:
+  D: 2
+  lims: [0.0, 1.0]
+  Re: 6000.0
+  tburn: 0.5
+  tsim: 5.0
+  savefreq: 50
+  ndns: 4096
+  nles: [64]
+  filters: ["FaceAverage()"]
+  icfunc: "(setup, psolver, rng) -> random_field(setup, T(0); kp=20, psolver, rng)"
+  method: "RKMethods.Wray3(; T)"
+  bodyforce: "(dim, x, y, t) -> (dim == 1) * 5 * sinpi(8 * y)"
+  issteadybodyforce: true
+  processors: "(; log = timelogger(; nupdate=100))"
+  Δt: 0.00005
+seeds:
+  dns: 123456
+  θ_start: 234
+  prior: 345
+  post: 456
+closure:
+  name: "cnn_backsol"
+  type: cnn
+  radii: [2, 2, 2, 2, 2]
+  channels: [24, 24, 24, 24, 2]
+  activations: ["tanh", "tanh", "tanh", "tanh", "identity"]
+  use_bias: [true, true, true, true, false]
+  rng: "Xoshiro(seeds.θ_start)"
+priori:
+  reuse: "cnn_project"
+  dotrain: true
+  nepoch: 50000
+  batchsize: 64
+  opt: "OptimiserChain(Adam(T(1.0e-3)), ClipGrad(0.1))"
+  do_plot: false
+  plot_train: false
+posteriori:
+  #reuse: "cnn_project"
+  dotrain: true
+  projectorders: "(ProjectOrder.Last, )"
+  nepoch: 1500
+  opt: "OptimiserChain(Adam(T(1.0e-4)), ClipGrad(0.01))"
+  nunroll: 5
+  nunroll_valid: 10
+  nsamples: 1
+  dt: 0.0001
+  do_plot: false
+  plot_train: false
+  sensealg: "BacksolveAdjoint()"

configs/snellius/cnn_interp.yaml

@@ -0,0 +1,54 @@
+docreatedata: true
+docomp: true
+ntrajectory: 8
+T: "Float32"
+dataproj: true
+params:
+  D: 2
+  lims: [0.0, 1.0]
+  Re: 6000.0
+  tburn: 0.5
+  tsim: 5.0
+  savefreq: 50
+  ndns: 4096
+  nles: [64]
+  filters: ["FaceAverage()"]
+  icfunc: "(setup, psolver, rng) -> random_field(setup, T(0); kp=20, psolver, rng)"
+  method: "RKMethods.Wray3(; T)"
+  bodyforce: "(dim, x, y, t) -> (dim == 1) * 5 * sinpi(8 * y)"
+  issteadybodyforce: true
+  processors: "(; log = timelogger(; nupdate=100))"
+  Δt: 0.00005
+seeds:
+  dns: 123456
+  θ_start: 234
+  prior: 345
+  post: 456
+closure:
+  name: "cnn_interp"
+  type: cnn
+  radii: [2, 2, 2, 2, 2]
+  channels: [24, 24, 24, 24, 2]
+  activations: ["tanh", "tanh", "tanh", "tanh", "identity"]
+  use_bias: [true, true, true, true, false]
+  rng: "Xoshiro(seeds.θ_start)"
+priori:
+  reuse: "cnn_project"
+  dotrain: true
+  nepoch: 50000
+  batchsize: 64
+  opt: "OptimiserChain(Adam(T(1.0e-3)), ClipGrad(0.1))"
+  do_plot: false
+  plot_train: false
+posteriori:
+  dotrain: true
+  projectorders: "(ProjectOrder.Last, )"
+  nepoch: 1500
+  opt: "OptimiserChain(Adam(T(1.0e-4)), ClipGrad(0.01))"
+  nunroll: 5
+  nunroll_valid: 10
+  nsamples: 1
+  dt: 0.0001
+  do_plot: false
+  plot_train: false
+  sensealg: "InterpolatingAdjoint()"

configs/snellius/cnn_lbfgs.yaml

@@ -0,0 +1,55 @@
+docreatedata: true
+docomp: true
+ntrajectory: 8
+T: "Float32"
+dataproj: true
+params:
+  D: 2
+  lims: [0.0, 1.0]
+  Re: 6000.0
+  tburn: 0.5
+  tsim: 5.0
+  savefreq: 50
+  ndns: 4096
+  nles: [64]
+  filters: ["FaceAverage()"]
+  icfunc: "(setup, psolver, rng) -> random_field(setup, T(0); kp=20, psolver, rng)"
+  method: "RKMethods.Wray3(; T)"
+  bodyforce: "(dim, x, y, t) -> (dim == 1) * 5 * sinpi(8 * y)"
+  issteadybodyforce: true
+  processors: "(; log = timelogger(; nupdate=100))"
+  Δt: 0.00005
+seeds:
+  dns: 123456
+  θ_start: 234
+  prior: 345
+  post: 456
+closure:
+  name: "cnn_lbfgs"
+  type: cnn
+  radii: [2, 2, 2, 2, 2]
+  channels: [24, 24, 24, 24, 2]
+  activations: ["tanh", "tanh", "tanh", "tanh", "identity"]
+  use_bias: [true, true, true, true, false]
+  rng: "Xoshiro(seeds.θ_start)"
+priori:
+  reuse: "cnn_project"
+  dotrain: true
+  nepoch: 50000
+  batchsize: 64
+  opt: "OptimiserChain(Adam(T(1.0e-3)), ClipGrad(0.1))"
+  do_plot: false
+  plot_train: false
+posteriori:
+  reuse: "cnn_project"
+  dotrain: true
+  projectorders: "(ProjectOrder.Last, )"
+  nepoch: 2000
+  opt: "Optim.LBFGS()"
+  nunroll: 5
+  nunroll_valid: 10
+  nsamples: 1
+  dt: 0.0001
+  do_plot: false
+  plot_train: false
+  sensealg: "InterpolatingAdjoint()"

extra_model_workflow.jl

@@ -259,6 +259,18 @@ end
 # Save parameters to disk after each run.
 # Plot training progress (for a validation data batch).
 
+# Check if it is asked to re-use the a-priori training from a different model
+if haskey(conf["priori"], "reuse")
+    reuse = conf["priori"]["reuse"]
+    @info "Reuse a-priori training from closure named: $reuse"
+    reusepriorfile(reuse, outdir, closure_name)
+end
+if haskey(conf["posteriori"], "reuse")
+    reuse = conf["posteriori"]["reuse"]
+    @info "Reuse a-posteriori training from closure named: $reuse"
+    reusepostfile(reuse, outdir, closure_name)
+end
+
 # Train
 @info "A priori training"
 for i = 1:ntrajectory

src/Benchmark.jl

@@ -112,6 +112,6 @@ export _convert_to_single_index,
     plot_epost_vs_t
 
 export compute_eprior, compute_epost, compute_t_prior_inference
-export reusepriorfile
+export reusepriorfile, reusepostfile
 
 end # module Benchmark

src/train.jl

@@ -68,6 +68,23 @@ function reusepriorfile(reuse, outdir, closure_name)
     end
 end
 
+function reusepostfile(reuse, outdir, closure_name)
+    reusepath = joinpath(outdir, "posttraining", reuse)
+    targetpath = joinpath(outdir, "posttraining", closure_name)
+    # If the reuse path exists, copy it to the target path
+    if ispath(reusepath)
+        @info "Reusing post training from $(reusepath) to $(targetpath)"
+        ispath(targetpath) || mkpath(targetpath)
+        for file in readdir(reusepath, join = true)
+            @info "Copying post training file $(file) to $(targetpath)"
+            cp(file, joinpath(targetpath, basename(file)); force = true)
+        end
+    else
+        @warn "Reuse path $(reusepath) does not exist. Not reusing post training."
+    end
+end
+
+
 "Load a-priori training results from correct file names."
 loadprior(outdir, closure_name, nles, filters) = map(
     splat((nles, Φ) -> load_object(getpriorfile(outdir, closure_name, nles, Φ))),