bigfeat: detector changed hook

.github/workflows/ci.yml

@@ -239,14 +239,16 @@ jobs:
         id:
           - kpp
           - eb-ep
+          - eb-en
           - eb-eg
           - eb-epc
           - eb-enc
           - eb-eftpi
         include:
-          # run all tests on ubuntu
+          # run selected tests on ubuntu
           - { id: kpp,      cmd: ./run-advanced-tests.sh }
           - { id: eb-ep,    cmd: ./run-eb-tests.sh -100 electronproton   }
+          - { id: eb-en,    cmd: ./run-eb-tests.sh -100 electronneutron  }
           - { id: eb-eg,    cmd: ./run-eb-tests.sh -100 electrongamma    }
           - { id: eb-epc,   cmd: ./run-eb-tests.sh -100 electronprotonC  }
           - { id: eb-enc,   cmd: ./run-eb-tests.sh -100 electronneutronC }

common-tools/clas-decay-tools/src/main/java/org/jlab/clas/decay/analysis/Analysis.java

@@ -67,7 +67,7 @@ public void setFieldsConfig(String fieldsConfig) {
 
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
         Reader reader ;
         this.FieldsConfig = this.getFieldsConfig();
         if (event.hasBank("RUN::config") == false) {
@@ -250,5 +250,7 @@ public double[] getBeamSpot(DataEvent event, IndexedTable beamPos) {
         }
         return xybeam;
     }
+
+    public void detectorChanged(int run) {}
 
 }

common-tools/clas-io/src/main/java/org/jlab/utils/JsonUtils.java

@@ -113,9 +113,7 @@ public static JsonObject Map2Json(Map<String,Object> map) {
                 ret.add(topKey,Map2Json((Map)entry.getValue()));
             }
             else {
-                if(entry.getValue() != null) {
-                    ret.add(topKey, entry.getValue().toString());
-                }
+                ret.add(topKey, entry.getValue() == null ? null : entry.getValue().toString());
             }
         }
         return ret;

common-tools/clas-reco/src/main/java/org/jlab/clas/reco/EngineProcessor.java

@@ -282,7 +282,7 @@ public void init(){
     public void processEvent(DataEvent event){
         for(Map.Entry<String,ReconstructionEngine> engine : this.processorEngines.entrySet()){
             try {
-                engine.getValue().filterEvent(event);
+                engine.getValue().processDataEvent(event);
             } catch (Exception e){
                 LOGGER.log(Level.SEVERE, "[Exception] >>>>> engine : {0}\n\n", engine.getKey());
                 e.printStackTrace();

common-tools/clas-reco/src/main/java/org/jlab/clas/reco/ReconstructionEngine.java

@@ -2,6 +2,7 @@
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -60,14 +61,22 @@ public abstract class ReconstructionEngine implements Engine {
     volatile boolean dropOutputBanks = false;
     private final Set<String> outputBanks = new HashSet<>();
 
+    private volatile List<Integer> runNumbers = new ArrayList<>();
+
     private boolean ignoreInvalidRunNumbers = true;
 
+    private int runNumberOverride = -1;
+
     volatile long triggerMask = 0xFFFFFFFFFFFFFFFFL;
 
-    String             engineName        = "UnknownEngine";
-    String             engineAuthor      = "N.T.";
-    String             engineVersion     = "0.0";
-    String             engineDescription = "CLARA Engine";
+    String engineName        = "UnknownEngine";
+    String engineAuthor      = "N.T.";
+    String engineVersion     = "0.0";
+    String engineDescription = "CLARA Engine";
+
+    abstract public boolean processDataEventUser(DataEvent event);
+    abstract public boolean init();
+    abstract public void detectorChanged(int runNumber);
 
     public ReconstructionEngine(String name, String author, String version){
         engineName    = name;
@@ -108,9 +117,6 @@ protected RawBank getRawBankReader(String bankName, OrderType... order) {
         return new RawDataBank(bankName, order);
     }
 
-    abstract public boolean processDataEvent(DataEvent event);
-    abstract public boolean init();
-
     /**
      * Use a map just to avoid name clash in ConstantsManager.
      * @param tables map of table names to #indices
@@ -187,6 +193,9 @@ public EngineData configure(EngineData ed) {
           engineDictionary = new SchemaFactory();
       LOGGER.log(Level.FINEST,"--- engine configuration is called " + this.getDescription());
       try {
+          if (this.getEngineConfigString("runNumberOverride")!=null) {
+              this.runNumberOverride = Integer.valueOf(this.getEngineConfigString("runNumberOverride"));
+          }
           if (this.getEngineConfigString("rawBankGroup")!=null) {
               this.rawBankOrders = RawBank.getFilterGroup(this.getEngineConfigString("rawBankGroup"));
           }
@@ -227,8 +236,7 @@ public EngineData configure(EngineData ed) {
     }
 
     protected String getStringConfigParameter(String jsonString,                                             
-            String key)  throws Exception {
-        Object js;
+                                              String key)  throws Exception {
         String variation = "";
         try {
             JSONObject base = new JSONObject(jsonString);
@@ -238,13 +246,6 @@ protected String getStringConfigParameter(String jsonString,
             } else {
                 LOGGER.log(Level.FINE,"[JSON]" + this.getName() + " **** warning **** does not contain key = " + key);
             }
-            /*
-            js = base.get(key);
-            if (js instanceof String) {
-                return (String) js;
-            } else {
-                throw new Exception("JSONObject[" +  "] not a string.");
-            }*/
         } catch (JSONException e) {
             throw new Exception(e.getMessage());
         }
@@ -345,17 +346,22 @@ public void dropBanks(DataEvent event) {
             }
         }
     }
-
-    public boolean checkRunNumber(DataEvent event) {
-        if (!this.ignoreInvalidRunNumbers) return true;
-        int run = 0;
-        if (event.hasBank("RUN::config")) {
-            run = event.getBank("RUN::config").getInt("run",0);
+
+    public synchronized boolean checkRunNumber(DataEvent event) {
+        int r = runNumberOverride;
+        if (r <= 0 && event.hasBank("RUN::config")) {
+            r = event.getBank("RUN::config").getInt("run",0);
         }
-        return run>0;
+        if (r > 0) {
+            if (this.runNumbers.isEmpty() || r != this.runNumbers.get(this.runNumbers.size()-1)) {
+                this.runNumbers.add(r);
+                this.detectorChanged(r);
+            }
+        }
+        return !this.ignoreInvalidRunNumbers || r>0;
     }
 
-    public void filterEvent(DataEvent dataEvent) {
+    public void processDataEvent(DataEvent dataEvent) {
         if (!this.wroteConfig) {
             this.wroteConfig = true;
             JsonUtils.extend(dataEvent, CONFIG_BANK_NAME, "json", this.generateConfig());
@@ -365,7 +371,7 @@ public void filterEvent(DataEvent dataEvent) {
         }
         if(this.applyTriggerMask(dataEvent)) {
             if (this.checkRunNumber(dataEvent)) {
-                this.processDataEvent(dataEvent);
+                this.processDataEventUser(dataEvent);
             }
         }        
     }
@@ -404,7 +410,7 @@ public EngineData execute(EngineData input) {
             }
 
             try {
-                this.filterEvent(dataEventHipo);
+                this.processDataEvent(dataEventHipo);
                 output.setData(mt, dataEventHipo.getHipoEvent());
             } catch (Exception e) {
                 String msg = String.format("Error processing input event%n%n%s", ClaraUtil.reportException(e));
@@ -519,16 +525,21 @@ public Reco(){
             super("a","b","c");
         }
         @Override
-        public boolean processDataEvent(DataEvent event) {
-            throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+        public boolean processDataEventUser(DataEvent event) {
+            throw new UnsupportedOperationException("Not supported yet.");
         }
 
         @Override
         public boolean init() {
-            throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+            throw new UnsupportedOperationException("Not supported yet.");
         }
-
-}
+
+        @Override
+        public void detectorChanged(int runNumber) {
+            throw new UnsupportedOperationException("Not supported yet.");
+        }
+    }
+
     public static void main(String[] args){
         System.setProperty("CLAS12DIR", "/Users/gavalian/Work/Software/project-3a.0.0/Distribution/clas12-offline-software/coatjava");
         try {
@@ -549,7 +560,6 @@ public static void main(String[] args){
                     "\"timestamp\":333\n" +
                     "}";
             System.out.println(json);
-            //json = "{ \"ccdb\":{\"run\":10,\"variation\":\"default\"}, \"variation\":\"cosmic\"}";
             Reco reco = new Reco();
             String variation =  reco.getStringConfigParameter(json, "variation");
             System.out.println(" Variation : " + variation);

common-tools/clas-reco/src/main/java/org/jlab/clas/reco/UberEngine.java

@@ -30,7 +30,7 @@ protected void add(ReconstructionEngine... e) {
     }
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
         throw new RuntimeException("UberEngine does not implement processDataEvent.");
     }
 
@@ -39,9 +39,9 @@ public boolean processDataEvent(DataEvent event) {
      * @param event 
      */
     @Override
-    public final void filterEvent(DataEvent event) {
+    public final void processDataEvent(DataEvent event) {
         for (ReconstructionEngine e : engines) 
-            e.filterEvent(event);
+            e.processDataEvent(event);
     }
 
     /**
@@ -55,6 +55,16 @@ public final boolean init() {
             if (!e.init()) ret = false;
         return ret;
     }
+
+    /**
+     * Run all engines' dettectorChanged methods.
+     * @param runNumber
+     */ 
+    @Override
+    public final void detectorChanged(int runNumber) {
+        for (ReconstructionEngine e : engines)
+            e.detectorChanged(runNumber);
+    }
 
     /**
      * Run all engines' configure methods.

common-tools/clas-reco/src/main/java/org/jlab/clas/service/PulseExtractorEngine.java

@@ -30,7 +30,7 @@ public boolean init() {
 	}
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
 
         // No CCDB table, hardcoded parameters in the extractor:
         //mode3.update(6, null, event, "BMT::wf", "BMT::adc");
@@ -50,4 +50,7 @@ public boolean processDataEvent(DataEvent event) {
         return true;
     }
 
+    @Override
+    public void detectorChanged(int runNumber) {}
+
 }

common-tools/swim-tools/src/main/java/org/jlab/clas/swimtools/MagFieldsEngine.java

@@ -162,7 +162,7 @@ private void loadTables() {
     }
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
         DataBank bank = event.getBank("RUN::config");
         // Load the constants
         // -------------------
@@ -195,4 +195,6 @@ public boolean init() {
         return true;
     }
 
+    @Override
+    public void detectorChanged(int runNumber) {}
 }

libexec/env.sh

@@ -2,6 +2,8 @@
 
 export CLAS12DIR=$(cd $(dirname ${BASH_SOURCE[0]:-$0})/.. && pwd -P)
 
+export PATH=$CLAS12DIR/bin:$PATH
+
 # Set default field maps (but do not override user's env):
 if [ -z "${COAT_MAGFIELD_TORUSMAP-}" ]; then
     export COAT_MAGFIELD_TORUSMAP=Symm_torus_r2501_phi16_z251_24Apr2018.dat

reconstruction/ai/src/main/java/org/jlab/service/ai/DCClsComboEngine.java

@@ -58,6 +58,9 @@ public DCClsComboEngine() {
         super("DCClsComboEngine","tongtong","1.0");
     }
 
+    @Override
+    public void detectorChanged(int run){}
+
     @Override
     public boolean init() {
         System.setProperty("ai.djl.pytorch.num_interop_threads", "1");
@@ -116,7 +119,7 @@ public boolean init() {
     }
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
 
         //// 6-cluster combo prediction
         // load clusters from bank HitBasedTrkg::Clusters         
@@ -537,4 +540,4 @@ public float[] processOutput(TranslatorContext ctx, NDList out) {
             public Batchifier getBatchifier() { return Batchifier.STACK; }
         };
     }        
-}
+}

reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java

@@ -79,7 +79,11 @@ public boolean init() {
     }
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public void detectorChanged(int run) {}
+
+    @Override
+    public boolean processDataEventUser(DataEvent event) {
+
         for (String bankName : BANK_NAMES) {
             if (!event.hasBank(bankName)) continue;
 

reconstruction/alert/src/main/java/org/jlab/service/ahdc/AHDCEngine.java

@@ -63,6 +63,11 @@ public boolean init(ModeTrackFinding m) {
         return init();
     }
 
+    @Override
+    public void detectorChanged(int run) {
+        // FIXME:  move geometry initialization here
+    }
+
     @Override
     public boolean init() {
 
@@ -109,7 +114,7 @@ else if (Objects.equals(this.getEngineConfigString("Mode"), ModeTrackFinding.CV_
     int Run = -1;
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
 
         if(event.hasBank("MC::Particle")) simulation = true;
 

reconstruction/alert/src/main/java/org/jlab/service/alert/ALERTEngine.java

@@ -85,6 +85,9 @@ public ALERTEngine() {
         super("ALERT", "whit,ouillon,pilleux", "0.1");
     }
 
+    @Override
+    public void detectorChanged(int run) {}
+
     /** 
      * ALERTEngine initialization.
      * Creates the RecoBankWriter and checks for various yaml flags.
@@ -120,7 +123,7 @@ public boolean init() {
      * </ul>
      */
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
 
         if (!event.hasBank("AHDC::adc")) 
             return false;

reconstruction/alert/src/main/java/org/jlab/service/atof/ATOFEngine.java

@@ -56,7 +56,8 @@ public Detector getATOF() {
     int Run = -1;
 
     @Override
-    public boolean processDataEvent(DataEvent event) {
+    public boolean processDataEventUser(DataEvent event) {
+
         if (!event.hasBank("RUN::config")) {
             return true;
         }
@@ -123,6 +124,11 @@ public boolean processDataEvent(DataEvent event) {
         return true;
     }
 
+    @Override
+    public void detectorChanged(int run) {
+        // FIXME:  move geometry initialization here
+    }
+
     @Override
     public boolean init() {
         rbc = new RecoBankWriter();