Skip to content

Commit cba570c

Browse files
baltzellbaltzell
authored
Cleanup clara decoder (#821)
* bugfix * add docs * add partial clear * comments * keep latest states * consilidate and rename clara decoder services * remove unnecessary confi * remove unnecessary config option * cleanup * add comments * add comment * keep 30 helicity states (RNG register size) * keep 60 helicity states (2*RNG register size) * keep a few more scaler readings too * cleanup * cleanup * propagate class name changes to example yamls * cleanup
1 parent c90777b commit cba570c

File tree

9 files changed

+93
-159
lines changed

9 files changed

+93
-159
lines changed

‎common-tools/clara-io/src/main/java/org/jlab/io/clara/EvioToHipoReader.java renamed to ‎common-tools/clara-io/src/main/java/org/jlab/io/clara/DecoderReader.java

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,11 @@
1414
import org.json.JSONObject;
1515

1616
/**
17-
*
17+
* Combined with DecoderWriter, a port of the standard "decoder" to CLARA.
18+
*
1819
* @author baltzell
1920
*/
20-
public class EvioToHipoReader extends AbstractEventReaderService<EvioSource> {
21-
22-
boolean collectGarbage = true; // for memory leak in CompactEvioReader
21+
public class DecoderReader extends AbstractEventReaderService<EvioSource> {
2322

2423
CLASDecoder4 decoder;
2524
private long maxEvents;
@@ -56,12 +55,14 @@ public ByteOrder readByteOrder() throws EventReaderException {
5655

5756
@Override
5857
public Object readEvent(int eventNumber) throws EventReaderException {
59-
if (eventNumber >= maxEvents) return null;
58+
if (eventNumber++ >= maxEvents) return null;
6059
try {
61-
ByteBuffer bb = reader.getEventBuffer(++eventNumber, true);
60+
ByteBuffer bb = reader.getEventBuffer(eventNumber, true);
6261
EvioDataEvent evio = new EvioDataEvent(bb.array(), readByteOrder());
6362
Event hipo = decoder.getDecodedEvent(evio, -1, eventNumber, torus, solenoid);
64-
if (eventNumber % 25000 == 0 && collectGarbage) System.gc();
63+
// FIXME: IIRC, this was added to (try to) address a memory leak in
64+
// CompactEvioReader, but it was ineffective and could/should be removed.
65+
if (eventNumber % 25000 == 0) System.gc();
6566
return hipo;
6667
} catch (EvioException e) {
6768
throw new EventReaderException(e);

‎common-tools/clara-io/src/main/java/org/jlab/io/clara/HipoToHipoTagWriter.java renamed to ‎common-tools/clara-io/src/main/java/org/jlab/io/clara/DecoderWriter.java

Lines changed: 63 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,26 @@
11
package org.jlab.io.clara;
22

3+
import java.io.File;
34
import java.nio.file.Path;
45
import java.util.TreeSet;
6+
import org.jlab.analysis.postprocess.Processor;
57
import org.jlab.clara.std.services.EventWriterException;
68
import org.jlab.detector.calib.utils.ConstantsManager;
79
import org.jlab.detector.decode.CLASDecoder4;
810
import org.jlab.detector.helicity.HelicitySequence;
11+
import org.jlab.detector.helicity.HelicitySequenceDelayed;
912
import org.jlab.detector.helicity.HelicityState;
1013
import org.jlab.detector.scalers.DaqScalersSequence;
1114
import org.jlab.jnp.hipo4.data.Bank;
1215
import org.jlab.jnp.hipo4.data.Event;
1316
import org.jlab.jnp.hipo4.data.SchemaFactory;
17+
import org.jlab.jnp.hipo4.io.HipoReader;
1418
import org.jlab.jnp.hipo4.io.HipoWriterSorted;
1519
import org.jlab.jnp.utils.file.FileUtils;
1620
import org.json.JSONObject;
1721

1822
/**
19-
* A port of the standard "decoder" to a CLARA I/O service.
23+
* Combined with DecoderReader, a port of the standard "decoder" to CLARA.
2024
*
2125
* 1. Converts EVIO to HIPO, translation tables, pulse extraction
2226
* 2. Copies special banks on-the-fly to new tag-1 events
@@ -25,22 +29,21 @@
2529
*
2630
* @author baltzell
2731
*/
28-
public class HipoToHipoTagWriter extends HipoToHipoWriter {
32+
public class DecoderWriter extends HipoToHipoWriter {
2933

30-
int tag = 1;
31-
String[] bankNames = {"RUN::scaler","HEL::scaler","RAW::scaler","RAW::epics","HEL::flip","COAT::config"};
34+
static final String[] TAG1BANKS = {"RUN::scaler","HEL::scaler","RAW::scaler","RAW::epics","HEL::flip","COAT::config"};
3235

33-
Bank[] banks;
36+
Bank[] tag1banks;
3437
Bank runConfig;
3538
Bank helicityAdc;
3639
ConstantsManager conman;
3740
TreeSet<HelicityState> helicities;
3841
DaqScalersSequence scalers;
3942
SchemaFactory fullSchema;
40-
int runNumber;
43+
boolean postprocess;
4144

4245
private void init(JSONObject opts) {
43-
runNumber = 0;
46+
postprocess = false;
4447
fullSchema = new SchemaFactory();
4548
fullSchema.initFromDirectory(FileUtils.getEnvironmentPath("CLAS12DIR","etc/bankdefs/hipo4"));
4649
runConfig = new Bank(fullSchema.getSchema("RUN::config"));
@@ -49,13 +52,12 @@ private void init(JSONObject opts) {
4952
scalers = new DaqScalersSequence(fullSchema);
5053
conman = new ConstantsManager();
5154
conman.init("/runcontrol/hwp","/runcontrol/helicity");
55+
if (opts.has("postprocess")) postprocess = opts.getBoolean("postprocess");
5256
if (opts.has("variation")) conman.setVariation(opts.getString("variation"));
5357
if (opts.has("timestamp")) conman.setTimeStamp(opts.getString("timestamp"));
54-
if (opts.has("tag")) tag = opts.getInt("tag");
55-
if (opts.has("banks")) bankNames = opts.getString("banks").split(",");
56-
banks = new Bank[bankNames.length];
57-
for (int i=0; i<banks.length; ++i)
58-
banks[i] = new Bank(fullSchema.getSchema(bankNames[i]));
58+
tag1banks = new Bank[TAG1BANKS.length];
59+
for (int i=0; i<tag1banks.length; ++i)
60+
tag1banks[i] = new Bank(fullSchema.getSchema(TAG1BANKS[i]));
5961
}
6062

6163
@Override
@@ -71,31 +73,70 @@ protected HipoWriterSorted createWriter(Path file, JSONObject opts) throws Event
7173
}
7274
}
7375

76+
/**
77+
* In addition to writing the incoming event, copies all tag-1 banks to new
78+
* tag-1 events and writes them, and stores helicity/scaler readings for later.
79+
* @param event
80+
* @throws EventWriterException
81+
*/
7482
@Override
7583
protected void writeEvent(Object event) throws EventWriterException {
7684
scalers.add((Event)event);
7785
((Event)event).read(runConfig);
78-
if (runConfig.getRows() > 0) {
79-
int r = runConfig.getInt("run",0);
80-
if (r > 0) runNumber = r;
81-
}
8286
((Event)event).read(helicityAdc);
8387
helicities.add(HelicityState.createFromFadcBank(helicityAdc, runConfig, conman));
84-
Event t = CLASDecoder4.createTaggedEvent((Event)event, runConfig, banks);
85-
if (!t.isEmpty()) writer.addEvent(t, tag);
88+
Event t = CLASDecoder4.createTaggedEvent((Event)event, runConfig, tag1banks);
89+
if (!t.isEmpty()) writer.addEvent(t, 1);
8690
super.writeEvent(event);
8791
}
8892

93+
/**
94+
* In addition to closing the writer, creates and writes tag-1 events with
95+
* HEL::flip bnks and clears old scaler/helicity readings.
96+
*/
8997
@Override
9098
protected void closeWriter() {
9199
HelicitySequence.writeFlips(fullSchema, writer, helicities);
92-
helicities.clear();
93-
scalers.clear();
94100
super.closeWriter();
101+
if (postprocess) postprocess();
102+
// keep the latest helicity/scaler reading for the next file:
103+
while (helicities.size() > 60) helicities.pollFirst();
104+
scalers.clear(10);
105+
}
106+
107+
private int getRunNumber() {
108+
Event e = new Event();
109+
HipoReader r = new HipoReader();
110+
r.open(filename);
111+
while (r.hasNext()) {
112+
r.nextEvent(e);
113+
e.read(runConfig);
114+
if (runConfig.getRows()>0 && runConfig.getInt("run",0)>0)
115+
return runConfig.getInt("run",0);
116+
}
117+
return 0;
95118
}
96119

97-
protected void closeRawWriter() {
98-
super.closeWriter();
120+
/**
121+
* Copy helicity/charge tag-1 information to all events.
122+
*/
123+
private void postprocess() {
124+
int d = conman.getConstants(getRunNumber(), "/runcontrol/helicity").getIntValue("delay",0,0,0);
125+
HelicitySequenceDelayed h = new HelicitySequenceDelayed(d);
126+
h.addStream(helicities);
127+
Processor p = new Processor(fullSchema, h, scalers);
128+
HipoReader r = new HipoReader();
129+
r.open(filename);
130+
Event e = new Event();
131+
writer.open("pp_"+filename);
132+
while (r.hasNext()) {
133+
r.nextEvent(e);
134+
p.processEvent(e);
135+
HipoToHipoWriter.writeEvent(writer, e, schemaBankList);
136+
}
137+
writer.close();
138+
new File(filename).delete();
139+
new File("pp_"+filename).renameTo(new File(filename));
99140
}
100141

101142
}

‎common-tools/clara-io/src/main/java/org/jlab/io/clara/HipoToHipoPostWriter.java

Lines changed: 0 additions & 45 deletions
This file was deleted.

‎common-tools/clara-io/src/main/resources/org/jlab/io/clara/EvioToHipoReader.yaml renamed to ‎common-tools/clara-io/src/main/resources/org/jlab/io/clara/DecoderReader.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
2-
name: EvioToHipoReader
3-
engine: org.jlab.io.clara.EvioToHipoReader
2+
name: DecoderReader
3+
engine: org.jlab.io.clara.DecoderReader
44
type: java
55

66
author: Nathan Baltzell

‎common-tools/clara-io/src/main/resources/org/jlab/io/clara/HipoToHipoTagWriter.yaml renamed to ‎common-tools/clara-io/src/main/resources/org/jlab/io/clara/DecoderWriter.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
2-
name: HipoToHipoTagWriter
3-
engine: org.jlab.io.clara.HipoToHipoTagWriter
2+
name: DecoderWriter
3+
engine: org.jlab.io.clara.DecoderWriter
44
type: java
55

66
author: Nathan Baltzell

‎common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
import java.util.HashMap;
99
import java.util.List;
1010
import java.util.Map;
11-
import java.util.logging.Logger;
1211
import org.jlab.logging.DefaultLogger;
1312

1413
import org.jlab.jnp.hipo4.data.Bank;
@@ -37,7 +36,6 @@ public class Processor {
3736

3837
private final String outputPrefix = "tmp_";
3938

40-
private boolean initialized;
4139
private ConstantsManager conman = null;
4240
private SchemaFactory schemaFactory = null;
4341
private DaqScalersSequence chargeSequence = null;
@@ -165,7 +163,6 @@ private void processEventScalers(Bank runConfig, Bank recEvent) {
165163
* @param e
166164
*/
167165
public void processEvent(DataEvent e) {
168-
if (!initialized) return;
169166
if (!e.hasBank("RUN::config")) return;
170167
if (!e.hasBank("REC::Event")) return;
171168
DataBank runConfig = e.getBank("RUN::config");
@@ -182,7 +179,6 @@ public void processEvent(DataEvent e) {
182179
* @param e
183180
*/
184181
public void processEvent(Event e) {
185-
if (!initialized) return;
186182
if (!e.hasBank(schemaFactory.getSchema("RUN::config"))) return;
187183
if (!e.hasBank(schemaFactory.getSchema("REC::Event"))) return;
188184
Bank runConfig = new Bank(schemaFactory.getSchema("RUN::config"));

‎common-tools/clas-detector/src/main/java/org/jlab/detector/scalers/DaqScalersSequence.java

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,12 @@
66
import java.util.Collections;
77
import java.util.Comparator;
88
import java.util.logging.Logger;
9-
import org.jlab.detector.calib.utils.ConstantsManager;
109

1110
import org.jlab.jnp.hipo4.io.HipoReader;
1211
import org.jlab.jnp.hipo4.data.Event;
1312
import org.jlab.jnp.hipo4.data.Bank;
1413
import org.jlab.jnp.hipo4.data.SchemaFactory;
15-
import org.jlab.utils.groups.IndexedTable;
14+
import org.jlab.detector.calib.utils.ConstantsManager;
1615

1716
/**
1817
* For easy access to most recent scaler readout for any given event.
@@ -105,18 +104,29 @@ protected int findIndex(long timestamp) {
105104

106105
public DaqScalersSequence(SchemaFactory schema) {
107106
runConfigBank = new Bank(schema.getSchema("RUN::config"));
108-
runScalerBank=new Bank(schema.getSchema("RUN::scaler"));
107+
runScalerBank = new Bank(schema.getSchema("RUN::scaler"));
109108
}
110109

111110
public DaqScalersSequence(List<DaqScalers> inputScalers) {
112111
for (DaqScalers inputScaler : inputScalers)
113112
this.add(inputScaler);
114113
}
115114

115+
/**
116+
* remove all readouts from the sequence
117+
*/
116118
public void clear() {
117119
scalers.clear();
118120
}
119-
121+
122+
/**
123+
* remove all but the latest readouts from the sequence
124+
* @param keep the number of readouts to keep
125+
*/
126+
public void clear(int keep) {
127+
while (scalers.size() > keep) scalers.remove(0);
128+
}
129+
120130
protected boolean add(DaqScalers ds) {
121131
if (this.scalers.isEmpty()) {
122132
this.scalers.add(ds);

‎etc/services/evio2hipo.yaml renamed to ‎etc/services/decode.yaml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
io-services:
22
reader:
3-
class: org.jlab.io.clara.EvioToHipoReader
4-
name: EvioToHipoReader
3+
class: org.jlab.io.clara.DecoderReader
4+
name: DecoderReader
55
writer:
6-
class: org.jlab.io.clara.HipoToHipoTagWriter
7-
name: HipoToHipoTagWriter
6+
class: org.jlab.io.clara.DecoderWriter
7+
name: DecoderWriter
88
services:
99
# FIXME: replace with dummy/useful service
1010
- class: org.jlab.service.eb.EBTBEngine

0 commit comments

Comments
 (0)