diff --git a/src/examples/hisqhmc_h.nim b/src/examples/hisqhmc_h.nim
new file mode 100644
index 0000000..cc5e2c7
--- /dev/null
+++ b/src/examples/hisqhmc_h.nim
@@ -0,0 +1,716 @@
+# A bit of a lazy Hasenbusch "upgrade" to hisqhmc.nim
+import qex
+import mdevolve
+import times,macros,json,parseopt
+import strformat,strutils,streams,os
+import gauge
+import layout
+import gauge/[hisqsmear]
+import algorithms/[integrator]
+import physics/[qcdTypes,stagSolve]
+
+export integrator
+
+const
+ ReversibilityCheck {.booldefine.} = false
+ ReadInputs {.booldefine.} = false
+ WriteGauge {.booldefine.} = false
+ ReadRNG {.booldefine.} = false
+ WriteRNG {.booldefine.} = false
+
+const banner = """
+|---------------------------------------------------------------|
+ Quantum EXpressions (QEX)
+
+ QEX authors: James Osborn & Xiao-Yong Jin
+ HISQ HMC authors:
+ - Curtis Taylor Peterson [C.T.P.] (Michigan State University)
+ - James Osborn (Argonne National Laboratory)
+ - Xiao-Yong Jin (Argonne National Laboratory)
+ QEX GitHub: https://github.com/jcosborn/qex
+ C.T.P. email: curtistaylorpetersonwork@gmail.com
+ cite: Proceedings of Science (PoS) LATTICE2016 (2017) 271
+|---------------------------------------------------------------|
+"""
+
+const
+ fileMd = "\ngenerated by QEX\n"
+ recordMd = "\nRNG field\n"
+
+const
+ ActionCGTol = 1e-20
+ ForceCGTol = 1e-12
+ ActionMaxCGIter = 10000
+ ForceMaxCGIter = 10000
+ ActionCGVerbosity = 1
+ ForceCGVerbosity = 1
+
+# Eqn. (A2) of arXiv:1004.0342; u0 = 1.0
+const
+ Cp = 1.0
+ Cr = -1.0/20.0
+
+let
+ # Read from file if ReadInputs = true
+ defaultInputs = %* {
+ "lattice-geometry": [8,8,8,8],
+ "hmc": {
+ "trajectory-length": 1.0,
+ "serial-rng": "milc",
+ "parallel-rng": "milc",
+ "serial-seed": 123456789,
+ "parallel-seed": 123456789,
+ "gauge-start": "cold"
+ },
+ "action": {
+ "beta": 12.0,
+ "mass": 0.1,
+ "hasenbusch-mass": 0.6,
+ "lepage": 0.0,
+ "naik": 1.0,
+ "boundary-conditions": "pppa"
+ },
+ "gauge": {
+ "integrator": "2MN",
+ "steps": 15
+ },
+ "fermion": {
+ "integrator": "2MN",
+ "steps": 15
+ }
+ }
+
+type
+ RNGType = enum MILC,MRG
+
+type
+ RNGRoot {.inheritable.} = object
+ generator*: RNGType
+ seed*: uint64
+ ParallelRNG* = object of RNGRoot
+ milc: typeof(Field[1,RngMilc6])
+ mrg: typeof(Field[1,MRG32k3a])
+ SerialRNG* = object of RNGRoot
+ milc: RngMilc6
+ mrg: MRG32k3a
+
+type
+ HisqHMCRoot[U] {.inheritable.} = object of RootObj
+ tau: float
+ hi,hf: float
+ trajs: int
+ integrator: ParIntegrator
+ srng: SerialRNG
+ prng: ParallelRNG
+ p,f: seq[U]
+ HisqHMC*[U,F,F0] = ref object of HisqHmcRoot[U]
+ beta,mass,hmass: float
+ bcs: string
+ u,u0: seq[U]
+ su,sul: seq[U]
+ phi,hphi: F
+ gc: GaugeActionCoeffs
+ stag: Staggered[U,F0]
+ params: HisqCoefs
+ spa,spf: SolverParams
+ perf: PerfInfo
+
+template UU(lo: Layout): untyped =
+ type(lo.ColorMatrix())
+template FF(lo: Layout): untyped =
+ type(lo.ColorVector())
+template FF0(lo: Layout): untyped =
+ type(lo.ColorVector()[0])
+
+proc reunit(g: auto) =
+ tic()
+ threads:
+ let d = g.checkSU
+ threadBarrier()
+ echo "unitary deviation avg: ",d.avg," max: ",d.max
+ g.projectSU
+ threadBarrier()
+ let dd = g.checkSU
+ echo "new unitary deviation avg: ",dd.avg," max: ",dd.max
+ toc("reunit")
+
+#[ For construction of RNG objects ]#
+
+proc new(
+ self: var RNGRoot;
+ generator: string;
+ seed: uint64
+ ) =
+ case generator:
+ of "milc","MILC","RngMilc6": self.generator = MILC
+ of "mrg","MRG","MRG32k3a": self.generator = MRG
+ else: qexError generator, " not supported"
+ self.seed = seed
+
+proc newParallelRNG*(
+ l: Layout;
+ generator: string;
+ seed: int;
+ ): ParallelRNG =
+ new(result, generator, uint64(seed))
+ case result.generator:
+ of MILC: result.milc = l.newRNGField(RngMilc6, result.seed)
+ of MRG: result.mrg = l.newRNGField(MRG32k3a, result.seed)
+
+proc seed(self: var SerialRNG) =
+ case self.generator:
+ of MILC: self.milc.seed(self.seed,987654321)
+ of MRG: self.mrg.seed(self.seed,987654321)
+
+proc newSerialRNG*(generator: string; seed: int): SerialRng =
+ new(result, generator, uint64(seed))
+ seed(result)
+
+#[ For construction of HisqHMC object ]#
+
+proc readJSON(fn: string): JsonNode = fn.parseFile
+
+proc readCMD: JsonNode =
+ var cmd = initOptParser()
+ result = parseJson("{}")
+ while true:
+ cmd.next()
+ case cmd.kind:
+ of cmdShortOption,cmdLongOption,cmdArgument:
+ try: result[cmd.key] = %* parseInt(cmd.val)
+ except ValueError:
+ try: result[cmd.key] = %* parseFloat(cmd.val)
+ except ValueError: result[cmd.key] = %* cmd.val
+ of cmdEnd: break
+
+proc newSolverParams(r2req: float; maxits,verbosity: int): auto =
+ result = initSolverParams()
+ result.r2req = r2req
+ result.maxits = maxits
+ result.verbosity = verbosity
+
+proc newSolverParams(info: JsonNode; af: string): auto =
+ let
+ r2 = case info["fermion"].hasKey("r2-" & af)
+ of true: info["fermion"]["r2-" & af].getFloat()
+ of false: (if af == "action": ActionCGTol else: ForceCGTol)
+ maxits = case info["fermion"].hasKey("maxits-" & af)
+ of true: info["fermion"]["maxits-" & af].getInt()
+ of false: (if af == "action": ActionMaxCGIter else: ForceMaxCGIter)
+ verbosity = case info["fermion"].hasKey("solver-verbosity-" & af)
+ of true: info["fermion"]["solver-verbosity-" & af].getInt()
+ of false: (if af == "action": ActionCGVerbosity else: ForceCGVerbosity)
+ result = newSolverParams(r2,maxits,verbosity)
+
+proc newHISQ(info: JsonNode): auto =
+ result = newHISQ(
+ info["action"]["lepage"].getFloat(),
+ info["action"]["naik"].getFloat()
+ )
+
+proc newSerialRNG(info: JsonNode): auto =
+ result = newSerialRNG(
+ info["hmc"]["serial-rng"].getStr(),
+ info["hmc"]["serial-seed"].getInt()
+ )
+
+proc newParallelRNG(lo: Layout; info: JsonNode): auto =
+ result = lo.newParallelRNG(
+ info["hmc"]["parallel-rng"].getStr(),
+ info["hmc"]["parallel-seed"].getInt()
+ )
+
+proc readGauge(u: auto; fn: string) =
+ if fileExists(fn):
+ if 0 != u.loadGauge(fn): qexError "unable to read " & fn
+ else: reunit(u)
+
+proc getIntSeq(input: JsonNode): seq[int] =
+ result = newSeq[int]()
+ for elem in input.getElems(): result.add elem.getInt()
+
+proc readParallelRNG(self: var HisqHMC; cmd: JsonNode) =
+ let tag = "parallel-rng-filename"
+ if cmd.hasKey(tag): self.prng.readRNG(cmd[tag].getStr())
+
+proc readSerialRNG(self: var HisqHMC; cmd: JsonNode) =
+ let tag = "serial-rng-filename"
+ if cmd.hasKey(tag): self.srng.readRNG(cmd[tag].getStr())
+
+proc setIntegrator(info: JsonNode; field: string): IntegratorProc =
+ result = toIntegratorProc(info[field]["integrator"].getStr())
+
+proc `$`*(self: HisqHMC): string =
+ let
+ params = (
+ trajectory_length: self.tau,
+ number_of_trajectories: self.trajs,
+ serial_RNG: self.srng.generator,
+ parallel_RNG: self.prng.generator,
+ beta: self.beta,
+ mass: self.mass,
+ hasenbusch_mass: self.hmass,
+ cp: Cp,
+ cr: Cr,
+ boundary_conditions: self.bcs
+ )
+ for tag,val in params.fieldPairs:
+ result = result & tag.replace("_"," ") & ": " & $(val) & "\n"
+ result = result & $(self.params)
+
+template newHisqHMC*(build: untyped): auto =
+ let
+ cmd = readCMD()
+ info = case cmd.hasKey("json-path")
+ of true: readJSON(cmd["json-path"].getStr())
+ of false: defaultInputs
+ lo = newLayout(info["lattice-geometry"].getIntSeq())
+ fermionSteps {.inject.} = info["fermion"]["steps"].getInt()
+ gaugeSteps {.inject.} = info["gauge"]["steps"].getInt()
+ fermionIntegrator {.inject.} = info.setIntegrator("fermion")
+ gaugeIntegrator {.inject.} = info.setIntegrator("gauge")
+ var
+ integrator {.inject.}: ParIntegrator
+ hisq {.inject.} = HisqHMC[lo.UU,lo.FF,lo.FF0]()
+
+ # Prepare HMC
+ (
+ hisq.tau,
+ hisq.trajs,
+ hisq.srng,
+ hisq.prng,
+ hisq.p,
+ hisq.f
+ ) = (
+ info["hmc"]["trajectory-length"].getFloat(),
+ (if cmd.hasKey("ntraj"): cmd["ntraj"].getInt() else: 1),
+ newSerialRNG(info),
+ lo.newParallelRNG(info),
+ lo.newGauge(),
+ lo.newGauge()
+ )
+ hisq.readParallelRNG(cmd)
+ hisq.readSerialRNG(cmd)
+
+ # Prepare HISQ
+ let beta = info["action"]["beta"].getFloat()
+ (
+ hisq.beta,
+ hisq.mass,
+ hisq.hmass,
+ hisq.bcs,
+ hisq.params,
+ hisq.spa,
+ hisq.spf,
+ hisq.gc
+ ) = (
+ beta,
+ info["action"]["mass"].getFloat(),
+ info["action"]["hasenbusch-mass"].getFloat(),
+ info["action"]["boundary-conditions"].getStr(),
+ newHISQ(info),
+ newSolverParams(info,"action"),
+ newSolverParams(info,"force"),
+ GaugeActionCoeffs(plaq: beta*Cp, rect: beta*Cr)
+ )
+
+ # Prepare fields
+ (
+ hisq.u,
+ hisq.u0,
+ hisq.su,
+ hisq.sul,
+ hisq.phi,
+ hisq.hphi
+ ) = (
+ lo.newGauge(),
+ lo.newGauge(),
+ lo.newGauge(),
+ lo.newGauge(),
+ lo.ColorVector(),
+ lo.ColorVector()
+ )
+ case info["hmc"]["gauge-start"].getStr():
+ of "cold","frozen","symmetric": hisq.u.unit()
+ of "hot","random":
+ hisq.prng.random(hisq.u)
+ hisq.u.reunit()
+ else: hisq.u.readGauge(cmd["gauge-filename"].getStr())
+ hisq.stag = newStag3(hisq.su,hisq.sul)
+
+ # Execute user commands and return result
+ build
+ hisq.integrator = integrator
+ hisq
+
+#[ Everything else... ]#
+
+proc uniform*(self: var SerialRNG): float32 =
+ case self.generator:
+ of MILC: result = self.milc.uniform()
+ of MRG: result = self.mrg.uniform()
+
+proc readRNG*(self: var SerialRNG; fn: string) =
+ var file = newFileStream(fn, fmRead)
+ if file.isNil: qexError "Was not able to read ", fn, ". Exiting."
+ else:
+ case self.generator:
+ of MILC: discard file.readData(self.milc.addr, self.milc.sizeof)
+ of MRG: discard file.readData(self.mrg.addr, self.mrg.sizeof)
+
+proc writeRNG*(self: var SerialRNG; fn: string) =
+ var file = newFileStream(fn, fmWrite)
+ if file.isNil: qexError "Unable to write to ", fn, ". Exiting."
+ else:
+ case self.generator:
+ of MILC: file.write self.milc
+ of MRG: file.write self.mrg
+ file.flush
+
+proc random*(self: var ParallelRNG; u: auto) =
+ case self.generator:
+ of MILC: u.random(self.milc)
+ of MRG: u.random(self.mrg)
+
+proc warm*(self: var ParallelRNG; u: auto) =
+ case self.generator:
+ of MILC: warm(u, 0.5, self.milc)
+ of MRG: warm(u, 0.5, self.mrg)
+
+proc readRNG*(self: var ParallelRNG; filename: string) =
+ case self.generator:
+ of MILC:
+ var reader = self.milc.l.newReader(filename)
+ reader.read(self.milc)
+ reader.close()
+ of MRG: qexError "MRG32k3a not currently supported for IO"
+
+proc writeRNG*(self: var ParallelRNG; filename: string) =
+ case self.generator:
+ of MILC:
+ var writer = self.milc.l.newWriter(filename, fileMd)
+ writer.write(self.milc, recordMd)
+ writer.close()
+ of MRG: qexError "MRG32k3a not currently supported for IO"
+
+proc randomTAHGaussian(lu: auto; pRNG: auto) =
+ threads:
+ for mu in 0..