From 9b09c0282d308cff4b1e35678cf570e68dc73478 Mon Sep 17 00:00:00 2001 From: tongtongcao Date: Thu, 12 Mar 2026 12:07:56 -0400 Subject: [PATCH 01/14] develop engine for CVT denoising --- reconstruction/cvt/pom.xml | 9 + .../rec/cvt/services/CVTDenoiseEngine.java | 533 ++++++++++++++++++ 2 files changed, 542 insertions(+) create mode 100644 reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java diff --git a/reconstruction/cvt/pom.xml b/reconstruction/cvt/pom.xml index 50e7eecd3a..7a9db52551 100644 --- a/reconstruction/cvt/pom.xml +++ b/reconstruction/cvt/pom.xml @@ -79,6 +79,11 @@ clas-geometry 14.1.2-SNAPSHOT + + org.jlab.clas12.detector + clas12detector-ai + 13.7.1-SNAPSHOT + org.jlab.clas12.detector clas12detector-eb @@ -95,6 +100,10 @@ junit junit + + ai.djl + api + diff --git a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java new file mode 100644 index 0000000000..70af041ea8 --- /dev/null +++ b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java @@ -0,0 +1,533 @@ +package org.jlab.rec.cvt.services; + +import ai.djl.MalformedModelException; +import java.nio.file.Paths; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.Shape; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; +import ai.djl.translate.Batchifier; +import ai.djl.inference.Predictor; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.translate.TranslateException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.jlab.clas.reco.ReconstructionEngine; +import org.jlab.io.base.DataBank; +import org.jlab.io.base.DataEvent; +import org.jlab.utils.system.ClasUtilsFile; +import org.jlab.geom.prim.Line3D; +import org.jlab.geom.prim.Arc3D; +import org.jlab.geom.prim.Point3D; +import org.jlab.rec.cvt.Geometry; +import org.jlab.clas.swimtools.Swim; +import org.jlab.rec.cvt.Constants; +import org.jlab.service.ai.PredictorPool; +import org.jlab.utils.groups.IndexedTable; + +/* ----------------------------------------------- + Input class + ----------------------------------------------- */ +class CVTInput { + + float[][] data; // [450][9] + float[] mask; // [450] + + public CVTInput(float[][] data, float[] mask) { + this.data = data; + this.mask = mask; + } +} + +/* ----------------------------------------------- + DJL Translator + ----------------------------------------------- */ +class CVTTranslator implements Translator { + + @Override + public NDList processInput(TranslatorContext ctx, CVTInput input) { + + NDManager manager = ctx.getNDManager(); + + NDArray x = manager.create(input.data).reshape(1, 450, 9); + NDArray mask = manager.create(input.mask).reshape(1, 450); + + return new NDList(x, mask); + } + + @Override + public float[][] processOutput(TranslatorContext ctx, NDList list) { + + NDArray output = list.singletonOrThrow(); // shape: (1, 450, C) or (1, 450) + output = output.squeeze(0); // remove batch dim -> (450, C) + + float[] flat = output.toFloatArray(); + long[] shape = output.getShape().getShape(); + + int dim0 = (int) shape[0]; + int dim1 = shape.length > 1 ? (int) shape[1] : 1; + + float[][] result = new float[dim0][dim1]; + + for(int i=0;i[] criterias; + private ZooModel[] model; + private PredictorPool[] predictors; + + public CVTDenoiseEngine() { + super("CVTDenoiseEngine","Tongtong","1.0"); + } + + + @Override + public boolean init() { + this.initConstantsTables(); + + criterias = new Criteria[NSECTIONS]; + model = new ZooModel[NSECTIONS]; + predictors = new PredictorPool[NSECTIONS]; + + initScaling(); // Initial minVals and maxVals + + System.setProperty("ai.djl.pytorch.num_interop_threads", "1"); + System.setProperty("ai.djl.pytorch.num_threads", "1"); + System.setProperty("ai.djl.pytorch.graph_optimizer", "false"); + + + // Load models and set predictor pools + for(int i = 0; i < NSECTIONS; i++){ + if (getEngineConfigString(CONF_THRESHOLDS[i]) != null) + thresholds[i] = Float.parseFloat(getEngineConfigString(CONF_THRESHOLDS[i])); + if (getEngineConfigString(CONF_MODEL_FILES[i]) != null) + modelFiles[i] = getEngineConfigString(CONF_MODEL_FILES[i]); + + try { + String modelPath = ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/data/nnet/cvtdn/" + modelFiles[i]); + + CVTTranslator translator = new CVTTranslator(); + criterias[i] = Criteria.builder() + .setTypes(CVTInput.class, float[][].class) + .optModelPath(Paths.get(modelPath)) + .optEngine("PyTorch") + .optTranslator(translator) + .optProgress(new ProgressBar()) + .build(); + + model[i] = criterias[i].loadModel(); + + int threads = Integer.parseInt(getEngineConfigString(CONF_THREADS,"16")); + predictors[i] = new PredictorPool(threads, model[i]); + } catch (NullPointerException | MalformedModelException | IOException | ModelNotFoundException ex) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, null, ex); + return false; + } + } + + return true; + } + + @Override + public boolean processDataEvent(DataEvent event) { + + Swim swimmer = new Swim(); + + int run = this.getRun(event); + + IndexedTable svtLorentz = this.getConstantsManager().getConstants(run, "/calibration/svt/lorentz_angle"); + IndexedTable bmtVoltage = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_voltage"); + + Geometry.getInstance().initialize(this.getConstantsManager().getVariation(), run, svtLorentz, bmtVoltage); + + if (!event.hasBank(BST_BANK)) return true; + if (!event.hasBank(BMT_BANK)) return true; + + DataBank bst_bank = event.getBank(BST_BANK); + DataBank bmt_bank = event.getBank(BMT_BANK); + + float[][][] x = new float[NSECTIONS][MAX_HITS][NFEATURES]; + float[][] mask = new float[NSECTIONS][MAX_HITS]; + + int[] nHits = {0, 0, 0}; + + Map[] maps = new HashMap[NSECTIONS]; + for (int i = 0; i < NSECTIONS; i++) { + maps[i] = new HashMap<>(); + } + + // Read BST bank and set input for models + int[][][] nHitsLayerSectorStrip_BST = new int[6][18][256]; // Number of hits in a strip + for(int i = 0; i < bst_bank.rows(); i++){ + int sector = bst_bank.getByte("sector", i); + int layer = bst_bank.getByte("layer", i); + int strip = bst_bank.getShort("component",i); + int order = bst_bank.getByte("order", i); + int adc = bst_bank.getInt("ADC", i); + + Hit hit = new Hit(layer, sector, strip, nHitsLayerSectorStrip_BST[layer-1][sector-1][strip-1]++); + + if(order == 0 || order == 10){ + Line3D line = Geometry.getInstance().getSVT().getStrip(layer, sector, strip); + float[] features = {strip, (float)(line.origin().x()/10.), (float)(line.end().x()/10.), (float)(line.origin().y()/10.), + (float)(line.end().y()/10.), (float)(line.origin().z()/10.), (float)(line.end().z()/10.), sector, layer}; // unit conversion from mm to cm for end points + + for(int f = 0; f < NFEATURES - 1; f++){ + float min = minVals[f][layer-1]; + float max = maxVals[f][layer-1]; + features[f] = (features[f] - min) / (max-min); + } + features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); + + List sectionList = getSectionList(layer, sector); + for(int section : sectionList){ + for(int f = 0; f < NFEATURES; f++){ + x[section - 1][nHits[section - 1]][f] = features[f]; + } + + mask[section - 1][nHits[section - 1]] = 1.0f; + + maps[section - 1].put(nHits[section - 1], hit); + + nHits[section - 1]++; + if(nHits[section - 1] == MAX_HITS) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); + return true; + } + } + } + } + + // Read BMT bank and set input for models + int[][][] nHitsLayerSectorStrip_BMT = new int[6][3][1152]; // Number of hits in a strip + for(int i = 0; i < bmt_bank.rows(); i++){ + int sector = bmt_bank.getByte("sector", i); + int layer = bmt_bank.getByte("layer", i); + int strip = bmt_bank.getShort("component",i); + int order = bmt_bank.getByte("order", i); + int adc = bst_bank.getInt("ADC", i); + + Hit hit = new Hit(layer+6, sector, strip, nHitsLayerSectorStrip_BMT[layer-1][sector-1][strip-1]++); + + if(order == 0 || order == 10){ + Point3D originPoint, endPoint; + int region = (layer + 1) / 2; // Get region number for a BMT layer; layers 1, 4, 6 for BMT-C layers, and layers 2, 3, 5 for BMT-Z + if(layer == 2 || layer ==3 || layer == 5) { + Line3D line = Geometry.getInstance().getBMT().getLCZstrip(region, sector, strip, swimmer); + originPoint = line.origin(); + endPoint = line.origin(); + } + else { + Arc3D arcLine = Geometry.getInstance().getBMT().getCstrip(region, sector, strip); + originPoint = arcLine.origin(); + endPoint = arcLine.end(); + } + + layer += 6; + float[] features = {strip, (float)(originPoint.x()/10.), (float)(endPoint.x()/10.), (float)(originPoint.y()/10.), + (float)(endPoint.y()/10.), (float)(originPoint.z()/10.), (float)(endPoint.z()/10.), sector, layer}; // unit conversion from mm to cm for end points + + for(int f = 0; f < NFEATURES - 1; f++){ + float min = minVals[f][layer-1]; + float max = maxVals[f][layer-1]; + features[f] = (features[f] - min) / (max-min); + } + features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); + + List sectionList = getSectionList(layer, sector); + for(int section : sectionList){ + for(int f = 0; f < NFEATURES; f++){ + x[section - 1][nHits[section - 1]][f] = features[f]; + } + + mask[section - 1][nHits[section - 1]] = 1.0f; + + maps[section - 1].put(nHits[section - 1], hit); + + nHits[section - 1]++; + if(nHits[section - 1] == MAX_HITS) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); + return true; + } + } + } + } + + // Apply models for prediction of hits + // Status for a hit with order of 0: + // 0: rejected + // 1: accepted by section 1 + // 2: accepted by section 2 + // 3: accepted by section 3 + // 12: accepted by sections 1 & 2; Note: a hit is shared by secton 1 & 2, and is accpcted by both section + // 23: accepted by sections 2 & 3; Note: a hit is shared by secton 2 & 3, and is accpcted by both section + int maxIndex_BST = Integer.MIN_VALUE; + for (int i = 0; i < nHitsLayerSectorStrip_BST.length; i++) { + for (int j = 0; j < nHitsLayerSectorStrip_BST[i].length; j++) { + for (int k = 0; k < nHitsLayerSectorStrip_BST[i][j].length; k++) { + if (nHitsLayerSectorStrip_BST[i][j][k] > maxIndex_BST) { + maxIndex_BST = nHitsLayerSectorStrip_BST[i][j][k]; + } + } + } + } + int maxIndex_BMT = Integer.MIN_VALUE; + for (int i = 0; i < nHitsLayerSectorStrip_BMT.length; i++) { + for (int j = 0; j < nHitsLayerSectorStrip_BMT[i].length; j++) { + for (int k = 0; k < nHitsLayerSectorStrip_BMT[i][j].length; k++) { + if (nHitsLayerSectorStrip_BMT[i][j][k] > maxIndex_BMT) { + maxIndex_BMT = nHitsLayerSectorStrip_BMT[i][j][k]; + } + } + } + } + int maxIndex = Math.max(maxIndex_BST, maxIndex_BMT); + + byte[][][][] statuses = new byte[NLAYERS][18][1152][maxIndex+1]; + for(int s = 0; s < NSECTIONS; s++){ + CVTInput input = new CVTInput(x[s],mask[s]); + try{ + Predictor predictor = predictors[s].take(); + + try{ + float[][] preds = predictor.predict(input); + for(int i = 0; i < nHits[s]; i++){ + Hit hit = maps[s].get(i); + byte status = statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()]; + if(preds[i][0] > thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] = (byte) (status * 10 + s + 1); + //if(statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] < 0 || statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] > 123) + // System.out.println(preds[i][0] + " " + s + " " + i + " " + hit.getLayer() + " " + hit.getSector() + " " + hit.getStrip() + " " + hit.getIndex() + " " + statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()]); + } + } finally { + predictors[s].put(predictor); + } + + } catch (TranslateException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + // Update BST and BMT banks to record predictions + updateBanks(bst_bank, bmt_bank, statuses); + event.removeBank(BST_BANK); + event.appendBank(bst_bank); + event.removeBank(BMT_BANK); + event.appendBank(bmt_bank); + + return true; + } + + // Scaling per layer + private void initScaling(){ + + for(int l=0;l<12;l++) minVals[0][l]=1f; + + int[] stripMax={256,256,256,256,256,256,896,640,640,1024,768,1152}; + for(int l=0;l<12;l++) maxVals[0][l]=stripMax[l]; + + float[] xyMin={-7,-8,-10,-10,-15,-15,-15,-18,-18,-23,-23,-23}; + float[] xyMax={7,8,10,10,15,15,15,18,18,23,23,23}; + + for(int f=1;f<=4;f++){ + for(int l=0;l<12;l++){ + minVals[f][l]=xyMin[l]; + maxVals[f][l]=xyMax[l]; + } + } + + float[] zMin={-25,-25,-22,-22,-18,-18,-18,-21,-21,-21,-21,-21}; + float[] zMax={25,25,22,22,18,18,21,21,21,25,25,25}; + + for(int f=5;f<=6;f++){ + for(int l=0;l<12;l++){ + minVals[f][l]=zMin[l]; + maxVals[f][l]=zMax[l]; + } + } + + float[] sectorMax={11,11,15,15,19,19,3,3,3,3,3,3}; + + for(int l=0;l<12;l++){ + minVals[7][l]=1f; + maxVals[7][l]=sectorMax[l]; + } + + /* + float[] timeMin={0,0,0,0,0,0,4,4,4,4,4,4}; + float[] timeMax={511,511,511,511,511,511,436,436,436,436,436,436}; + + for(int l=0;l<12;l++){ + minVals[8][l]=timeMin[l]; + maxVals[8][l]=timeMax[l]; + } + */ + } + + // Set sections based on layer and sector of hits + // Some SVT hits are shared by sections1&2 or sections2&3 + private List getSectionList(int layer, int sector) { + List sectionList = new ArrayList<>(); + + if (layer >= 1 && layer <= 2) { + if (sector >= 1 && sector <= 4) sectionList.add(1); + if (sector >= 4 && sector <= 8) sectionList.add(2); + if (sector >= 8 && sector <= 10) sectionList.add(3); + } + else if (layer >= 3 && layer <= 4) { + if (sector >= 1 && sector <= 6) sectionList.add(1); + if (sector >= 6 && sector <= 10) sectionList.add(2); + if (sector >= 10 && sector <= 14) sectionList.add(3); + } + else if (layer >= 5 && layer <= 6) { + if (sector >= 1 && sector <= 7) sectionList.add(1); + if (sector >= 7 && sector <= 13) sectionList.add(2); + if (sector >= 14 && sector <= 18) sectionList.add(3); + } + else { + if (sector == 1) sectionList.add(1); + if (sector == 2) sectionList.add(2); + if (sector == 3) sectionList.add(3); + } + + return sectionList; + } + + // -------- Update BST & BMT banks -------- + private void updateBanks(DataBank bst_bank, DataBank bmt_bank, byte[][][][] statuses) { + int[][][] nHitsLayerSectorStrip_BST = new int[6][18][256]; // Number of hits in a strip + for (int row=0; row Date: Mon, 16 Mar 2026 11:56:15 -0400 Subject: [PATCH 02/14] CVT Hits reconstruction stand-alone service. --- .../jlab/rec/cvt/services/CVTHitEngine.java | 447 ++++++++++++++++++ 1 file changed, 447 insertions(+) create mode 100644 reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTHitEngine.java diff --git a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTHitEngine.java b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTHitEngine.java new file mode 100644 index 0000000000..625002a192 --- /dev/null +++ b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTHitEngine.java @@ -0,0 +1,447 @@ +package org.jlab.rec.cvt.services; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.jlab.clas.reco.ReconstructionEngine; +import org.jlab.clas.swimtools.Swim; +import org.jlab.io.base.DataBank; +import org.jlab.io.base.DataEvent; +import org.jlab.rec.cvt.Constants; +import org.jlab.rec.cvt.Geometry; +import org.jlab.rec.cvt.banks.RecoBankWriter; +import org.jlab.rec.cvt.hit.Hit; +import org.jlab.utils.groups.IndexedTable; + +/** + * Service to return reconstructed HITS + * + * @author ziegler + * + */ +public class CVTHitEngine extends ReconstructionEngine { + + + /** + * @param docacutsum the docacutsum to set + */ + public void setDocacutsum(double docacutsum) { + this.docacutsum = docacutsum; + } + + private int Run = -1; + + private String svtHitBank; + private String bmtHitBank; + private String bankPrefix = ""; + + // run-time options + private int pid = 0; + private int kfIterations = 5; + private boolean kfFilterOn = true; + private boolean initFromMc = false; + + // yaml setting passed to Constants class + private boolean isCosmics = false; + private boolean svtOnly = false; + private String excludeLayers = null; + private String excludeBMTLayers = null; + private int removeRegion = 0; + private int beamSpotConstraint = 2; + private double beamSpotRadius = 0.3; + private String targetMaterial = ""; + private boolean elossPrecorrection = true; + private boolean svtSeeding = true; + private boolean timeCuts = true; + private boolean hvCuts = false; + public boolean useSVTTimingCuts = false; + public boolean removeOverlappingSeeds = true; + public boolean flagSeeds = true; + public boolean gemcIgnBMT0ADC = false; + public boolean KFfailRecovery = true; + public boolean KFfailRecovMisCls = true; + private String matrixLibrary = "EJML"; + private boolean useOnlyTruth = false; + private boolean useSVTLinkerSeeder = true; + private double docacut = 0.75; + private double docacutsum = 1.15; + private int svtmaxclussize = 100; + private int bmtcmaxclussize = 100; + private int bmtzmaxclussize = 100; + private double rcut = 120.0; + private double z0cut = 10; + + public CVTHitEngine(String name) { + super(name, "ziegler", "6.0"); + } + + public CVTHitEngine() { + super("CVTHitEngine", "ziegler", "6.0"); + } + + + @Override + public boolean init() { + this.loadConfiguration(); + Constants.getInstance().initialize(this.getName(), + isCosmics, + svtOnly, + excludeLayers, + excludeBMTLayers, + removeRegion, + beamSpotConstraint, + beamSpotRadius, + targetMaterial, + elossPrecorrection, + svtSeeding, + timeCuts, + hvCuts, + useSVTTimingCuts, + removeOverlappingSeeds, + flagSeeds, + gemcIgnBMT0ADC, + KFfailRecovery, + KFfailRecovMisCls, + matrixLibrary, + useOnlyTruth, + useSVTLinkerSeeder, + docacut, + docacutsum, + svtmaxclussize, + bmtcmaxclussize, + bmtzmaxclussize, + rcut, + z0cut); + + this.initConstantsTables(); + this.registerBanks(); + this.printConfiguration(); + return true; + } + + public final void setOutputBankPrefix(String prefix) { + this.bankPrefix = prefix; + } + + public void registerBanks() { + String prefix = bankPrefix; + if(Constants.getInstance().isCosmics) prefix = "Rec"; + this.setBmtHitBank("BMT" + prefix + "::Hits"); + this.setSvtHitBank("BST" + prefix + "::Hits"); + super.registerOutputBank(this.bmtHitBank); + super.registerOutputBank(this.svtHitBank); + } + + public int getRun(DataEvent event) { + + if (event.hasBank("RUN::config") == false) { + System.err.println("RUN CONDITIONS NOT READ!"); + return 0; + } + + DataBank bank = event.getBank("RUN::config"); + int run = bank.getInt("run", 0); + if(Constants.getInstance().seedingDebugMode) { + System.out.println("EVENT "+bank.getInt("event", 0)); + } + return run; + } + + public int getPid() { + return pid; + } + + public int getKfIterations() { + return kfIterations; + } + + public boolean isKfFilterOn() { + return kfFilterOn; + } + + public boolean isInitFromMc() { + return initFromMc; + } + + public boolean seedBeamSpot() { + return this.beamSpotConstraint>0; + } + + public boolean kfBeamSpot() { + return this.beamSpotConstraint==2; + } + + /** + * @return the docacut + */ + public double getDocacut() { + return docacut; + } + + /** + * @param docacut the docacut to set + */ + public void setDocacut(double docacut) { + this.docacut = docacut; + } + + /** + * @return the docacutsum + */ + public double getDocacutsum() { + return docacutsum; + } + + /** + * @return the svtmaxclussize + */ + public int getSvtmaxclussize() { + return svtmaxclussize; + } + + /** + * @param svtmaxclussize the svtmaxclussize to set + */ + public void setSvtmaxclussize(int svtmaxclussize) { + this.svtmaxclussize = svtmaxclussize; + } + + /** + * @return the bmtcmaxclussize + */ + public int getBmtcmaxclussize() { + return bmtcmaxclussize; + } + + /** + * @param bmtcmaxclussize the bmtcmaxclussize to set + */ + public void setBmtcmaxclussize(int bmtcmaxclussize) { + this.bmtcmaxclussize = bmtcmaxclussize; + } + + /** + * @return the bmtzmaxclussize + */ + public int getBmtzmaxclussize() { + return bmtzmaxclussize; + } + + /** + * @param bmtzmaxclussize the bmtzmaxclussize to set + */ + public void setBmtzmaxclussize(int bmtzmaxclussize) { + this.bmtzmaxclussize = bmtzmaxclussize; + } + + @Override + public boolean processDataEvent(DataEvent event) { + + Swim swimmer = new Swim(); + + int run = this.getRun(event); + + IndexedTable svtStatus = this.getConstantsManager().getConstants(run, "/calibration/svt/status"); + IndexedTable svtLorentz = this.getConstantsManager().getConstants(run, "/calibration/svt/lorentz_angle"); + IndexedTable bmtStatus = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_status"); + IndexedTable bmtTime = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_time"); + IndexedTable bmtVoltage = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_voltage"); + IndexedTable bmtStripVoltage = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_strip_voltage"); + IndexedTable bmtStripThreshold = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_strip_voltage_thresholds"); + IndexedTable beamPos = this.getConstantsManager().getConstants(run, "/geometry/beam/position"); + IndexedTable adcStatus = this.getConstantsManager().getConstants(run, "/calibration/svt/adcstatus"); + + Geometry.getInstance().initialize(this.getConstantsManager().getVariation(), run, svtLorentz, bmtVoltage); + + CVTReconstruction reco = new CVTReconstruction(swimmer); + + List> hits = reco.readHits(event, svtStatus, bmtStatus, bmtTime, + bmtStripVoltage, bmtStripThreshold, + adcStatus); + + + List banks = new ArrayList<>(); + + banks.add(RecoBankWriter.fillSVTHitBank(event, hits.get(0), this.getSvtHitBank())); + banks.add(RecoBankWriter.fillBMTHitBank(event, hits.get(1), this.getBmtHitBank())); + + event.appendBanks(banks.toArray(new DataBank[0])); + + + return true; + } + + + public void loadConfiguration() { + + // general (pass-independent) settings + if (this.getEngineConfigString("cosmics")!=null) + this.isCosmics = Boolean.valueOf(this.getEngineConfigString("cosmics")); + + if (this.getEngineConfigString("svtOnly")!=null) + this.svtOnly = Boolean.valueOf(this.getEngineConfigString("svtOnly")); + + if (this.getEngineConfigString("excludeLayers")!=null) + this.excludeLayers = this.getEngineConfigString("excludeLayers"); + + if (this.getEngineConfigString("excludeBMTLayers")!=null) + this.excludeBMTLayers = this.getEngineConfigString("excludeBMTLayers"); + + if (this.getEngineConfigString("removeRegion")!=null) + this.removeRegion = Integer.valueOf(this.getEngineConfigString("removeRegion")); + + if (this.getEngineConfigString("beamSpotConst")!=null) + this.beamSpotConstraint = Integer.valueOf(this.getEngineConfigString("beamSpotConst")); + + if (this.getEngineConfigString("beamSpotRadius")!=null) + this.beamSpotRadius = Double.valueOf(this.getEngineConfigString("beamSpotRadius")); + + if(this.getEngineConfigString("targetMat")!=null) + this.targetMaterial = this.getEngineConfigString("targetMat"); + + if(this.getEngineConfigString("elossPreCorrection")!=null) + this.elossPrecorrection = Boolean.parseBoolean(this.getEngineConfigString("elossPreCorrection")); + + if(this.getEngineConfigString("svtSeeding")!=null) + this.svtSeeding = Boolean.parseBoolean(this.getEngineConfigString("svtSeeding")); + + if(this.getEngineConfigString("timeCuts")!=null) + this.timeCuts = Boolean.parseBoolean(this.getEngineConfigString("timeCuts")); + + if(this.getEngineConfigString("hvCuts")!=null) + this.hvCuts = Boolean.parseBoolean(this.getEngineConfigString("hvCuts")); + + if(this.getEngineConfigString("useSVTTimingCuts")!=null) + this.useSVTTimingCuts = Boolean.parseBoolean(this.getEngineConfigString("useSVTTimingCuts")); + + if(this.getEngineConfigString("removeOverlappingSeeds")!=null) + this.removeOverlappingSeeds = Boolean.parseBoolean(this.getEngineConfigString("removeOverlappingSeeds")); + + if(this.getEngineConfigString("flagSeeds")!=null) + this.flagSeeds = Boolean.parseBoolean(this.getEngineConfigString("flagSeeds")); + + if(this.getEngineConfigString("gemcIgnBMT0ADC")!=null) + this.gemcIgnBMT0ADC = Boolean.parseBoolean(this.getEngineConfigString("gemcIgnBMT0ADC")); + + if(this.getEngineConfigString("KFfailRecovery")!=null) + this.KFfailRecovery = Boolean.parseBoolean(this.getEngineConfigString("KFfailRecovery")); + + if(this.getEngineConfigString("KFfailRecovMisCls")!=null) + this.KFfailRecovMisCls = Boolean.parseBoolean(this.getEngineConfigString("KFfailRecovMisCls")); + + if (this.getEngineConfigString("matLib")!=null) + this.matrixLibrary = this.getEngineConfigString("matLib"); + + // service dependent configuration settings + if(this.getEngineConfigString("elossPid")!=null) + this.pid = Integer.parseInt(this.getEngineConfigString("elossPid")); + + if (this.getEngineConfigString("kfFilterOn")!=null) + this.kfFilterOn = Boolean.valueOf(this.getEngineConfigString("kfFilterOn")); + + if (this.getEngineConfigString("initFromMC")!=null) + this.initFromMc = Boolean.valueOf(this.getEngineConfigString("initFromMC")); + + if (this.getEngineConfigString("useOnlyTruthHits")!=null) + this.useOnlyTruth = Boolean.valueOf(this.getEngineConfigString("useOnlyTruthHits")); + + if (this.getEngineConfigString("useSVTLinkerSeeder")!=null) + this.useSVTLinkerSeeder = Boolean.valueOf(this.getEngineConfigString("useSVTLinkerSeeder")); + + if (this.getEngineConfigString("kfIterations")!=null) + this.kfIterations = Integer.valueOf(this.getEngineConfigString("kfIterations")); + + if (this.getEngineConfigString("docacut")!=null) + this.setDocacut((double) Double.valueOf(this.getEngineConfigString("docacut"))); + + if (this.getEngineConfigString("docacutsum")!=null) + this.setDocacutsum((double) Double.valueOf(this.getEngineConfigString("docacutsum"))); + + if (this.getEngineConfigString("svtmaxclussize")!=null) + this.setSvtmaxclussize((int) Integer.valueOf(this.getEngineConfigString("svtmaxclussize"))); + + if (this.getEngineConfigString("bmtcmaxclussize")!=null) + this.setBmtcmaxclussize((int) Integer.valueOf(this.getEngineConfigString("bmtcmaxclussize"))); + + if (this.getEngineConfigString("bmtzmaxclussize")!=null) + this.setBmtzmaxclussize((int) Integer.valueOf(this.getEngineConfigString("bmtzmaxclussize"))); + + if (this.getEngineConfigString("rcut")!=null) + this.rcut = Double.valueOf(this.getEngineConfigString("rcut")); + + if (this.getEngineConfigString("z0cut")!=null) + this.z0cut = Double.valueOf(this.getEngineConfigString("z0cut")); + + } + + + public void initConstantsTables() { + String[] tables = new String[]{ + "/calibration/svt/status", + "/calibration/svt/lorentz_angle", + "/calibration/mvt/bmt_time", + "/calibration/mvt/bmt_status", + "/calibration/mvt/bmt_voltage", + "/calibration/mvt/bmt_strip_voltage", + "/calibration/mvt/bmt_strip_voltage_thresholds", + "/geometry/beam/position", + "/calibration/svt/adcstatus" + }; + requireConstants(Arrays.asList(tables)); + this.getConstantsManager().setVariation("default"); + } + + public void setSvtHitBank(String bstHitBank) { + this.svtHitBank = bstHitBank; + } + + public void setBmtHitBank(String bmtHitBank) { + this.bmtHitBank = bmtHitBank; + } + + public String getSvtHitBank() { + return svtHitBank; + } + + public String getBmtHitBank() { + return bmtHitBank; + } + + + public void printConfiguration() { + + System.out.println("["+this.getName()+"] run with cosmics setting set to "+Constants.getInstance().isCosmics); + System.out.println("["+this.getName()+"] run with SVT only set to "+Constants.getInstance().svtOnly); + if(this.excludeLayers!=null) + System.out.println("["+this.getName()+"] run with layers "+this.excludeLayers+" excluded in fit, based on yaml"); + if(this.excludeBMTLayers!=null) + System.out.println("["+this.getName()+"] run with BMT layers "+this.getEngineConfigString("excludeBMTLayers")+" excluded"); + if(this.removeRegion>0) + System.out.println("["+this.getName()+"] run with region "+this.getEngineConfigString("removeRegion")+" removed"); + System.out.println("["+this.getName()+"] run with beamSpotConst set to "+Constants.getInstance().beamSpotConstraint+ " (0=no-constraint, 1=seed only, 2=seed and KF)"); + System.out.println("["+this.getName()+"] run with beam spot size set to "+Constants.getInstance().getBeamRadius()); + System.out.println("["+this.getName()+"] Target material set to "+ Constants.getInstance().getTargetType()); + System.out.println("["+this.getName()+"] Pre-Eloss correction set to " + Constants.getInstance().preElossCorrection); + System.out.println("["+this.getName()+"] run SVT-based seeding set to "+ Constants.getInstance().svtSeeding); + System.out.println("["+this.getName()+"] run BMT timing cuts set to "+ Constants.getInstance().timeCuts); + System.out.println("["+this.getName()+"] run BMT HV masks "+ Constants.getInstance().bmtHVCuts); + System.out.println("["+this.getName()+"] run with matLib "+ Constants.getInstance().KFMatrixLibrary.toString() + " library"); + System.out.println("["+this.getName()+"] ELoss mass set for particle "+ pid); + System.out.println("["+this.getName()+"] run with Kalman-Filter status set to "+this.kfFilterOn); + System.out.println("["+this.getName()+"] initialize KF from true MC information "+this.initFromMc); + System.out.println("["+this.getName()+"] number of KF iterations set to "+this.kfIterations); + System.out.println("["+this.getName()+"] SLA doca cut "+this.docacut); + System.out.println("["+this.getName()+"] SLA docasum cut "+this.docacutsum); + System.out.println("["+this.getName()+"] max svt cluster size "+this.getSvtmaxclussize()); + System.out.println("["+this.getName()+"] max bmt-c cluster size "+this.getBmtcmaxclussize()); + System.out.println("["+this.getName()+"] max btm-z cluster size "+this.getBmtzmaxclussize()); + System.out.println("["+this.getName()+"] helix radius cut (mm) "+this.rcut); + System.out.println("["+this.getName()+"] z0 cut (mm from target edges) "+this.z0cut); + + + } + + + +} From 62461489fe4b91efad52e742ec7b264dec55b91b Mon Sep 17 00:00:00 2001 From: tongtongcao Date: Mon, 16 Mar 2026 17:56:52 -0400 Subject: [PATCH 03/14] update CVTDenoiseEngine with using BST::Hits and BMT::Hits as input --- etc/bankdefs/hipo4/bmt.json | 3 +- etc/bankdefs/hipo4/bst.json | 3 +- .../rec/cvt/services/CVTDenoiseEngine.java | 189 +++++++++--------- 3 files changed, 93 insertions(+), 102 deletions(-) diff --git a/etc/bankdefs/hipo4/bmt.json b/etc/bankdefs/hipo4/bmt.json index 616793bd1f..3d50008262 100644 --- a/etc/bankdefs/hipo4/bmt.json +++ b/etc/bankdefs/hipo4/bmt.json @@ -129,7 +129,8 @@ {"name":"trkingStat", "type":"B", "info":"tracking status"}, {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, - {"name":"status", "type":"B", "info":"status (0=good)"} + {"name":"status", "type":"B", "info":"status (0=good)"}, + {"name":"ai", "type":"B", "info":"3 bits corresponds to 3 sections; 1 = singal and 0 = noise"} ] }, { diff --git a/etc/bankdefs/hipo4/bst.json b/etc/bankdefs/hipo4/bst.json index 8d9c6c46ba..a45c3475b1 100644 --- a/etc/bankdefs/hipo4/bst.json +++ b/etc/bankdefs/hipo4/bst.json @@ -120,7 +120,8 @@ {"name":"trkingStat", "type":"B", "info":"tracking status"}, {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, - {"name":"status", "type":"B", "info":"status (0=good)"} + {"name":"status", "type":"B", "info":"status (0=good)"}, + {"name":"ai", "type":"B", "info":"3 bits corresponds to 3 sections; 1 = singal and 0 = noise"} ] }, { diff --git a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java index 70af041ea8..f860056984 100644 --- a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java +++ b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java @@ -104,7 +104,7 @@ class Hit { int layer; // 1 to 6 for SVT, and 7 to 12 for BMT int sector; int strip; - int index; // Index of hits in a strip in case that there are multiple hits in a strip + int index; // Index of hits in a strip, in case that there are multiple hits in a strip public Hit(int layer, int sector, int strip, int index) { this.layer = layer; @@ -140,14 +140,14 @@ public class CVTDenoiseEngine extends ReconstructionEngine { private static final int NFEATURES = 9; // Number of features for input of models // Inputs are from BST and BMT adc banks - final static String BST_BANK = "BST::adc"; - final static String BMT_BANK = "BMT::adc"; + final static String BST_BANK = "BST::Hits"; + final static String BMT_BANK = "BMT::Hits"; // Model files and thresholds for the three models; They could be set in yaml files final static String CONF_MODEL_FILES[] = {"modelFile1", "modelFile2", "modelFile3"}; final static String CONF_THRESHOLDS[] = {"threshold1", "threshold2", "threshold3"}; String[] modelFiles = {"classifier_torchscript_sector1_noCSWeight_weightInTraining.pt", "classifier_torchscript_sector2_noCSWeight_weightInTraining.pt", "classifier_torchscript_sector3_noCSWeight_weightInTraining.pt"}; - float[] thresholds = {0.025f, 0.025f, 0.025f}; // To be determined + float[] thresholds = {0.1f, 0.1f, 0.1f}; // To be determined // Number of threshold for predictor pools final static String CONF_THREADS = "threads"; @@ -245,41 +245,37 @@ public boolean processDataEvent(DataEvent event) { for(int i = 0; i < bst_bank.rows(); i++){ int sector = bst_bank.getByte("sector", i); int layer = bst_bank.getByte("layer", i); - int strip = bst_bank.getShort("component",i); - int order = bst_bank.getByte("order", i); - int adc = bst_bank.getInt("ADC", i); + int strip = bst_bank.getShort("strip",i); Hit hit = new Hit(layer, sector, strip, nHitsLayerSectorStrip_BST[layer-1][sector-1][strip-1]++); - if(order == 0 || order == 10){ - Line3D line = Geometry.getInstance().getSVT().getStrip(layer, sector, strip); - float[] features = {strip, (float)(line.origin().x()/10.), (float)(line.end().x()/10.), (float)(line.origin().y()/10.), - (float)(line.end().y()/10.), (float)(line.origin().z()/10.), (float)(line.end().z()/10.), sector, layer}; // unit conversion from mm to cm for end points - - for(int f = 0; f < NFEATURES - 1; f++){ - float min = minVals[f][layer-1]; - float max = maxVals[f][layer-1]; - features[f] = (features[f] - min) / (max-min); - } - features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); - - List sectionList = getSectionList(layer, sector); - for(int section : sectionList){ - for(int f = 0; f < NFEATURES; f++){ - x[section - 1][nHits[section - 1]][f] = features[f]; - } - - mask[section - 1][nHits[section - 1]] = 1.0f; - - maps[section - 1].put(nHits[section - 1], hit); - - nHits[section - 1]++; - if(nHits[section - 1] == MAX_HITS) { - Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); - return true; - } - } - } + Line3D line = Geometry.getInstance().getSVT().getStrip(layer, sector, strip); + float[] features = {strip, (float)(line.origin().x()/10.), (float)(line.end().x()/10.), (float)(line.origin().y()/10.), + (float)(line.end().y()/10.), (float)(line.origin().z()/10.), (float)(line.end().z()/10.), sector, layer}; // unit conversion from mm to cm for end points + + for(int f = 0; f < NFEATURES - 1; f++){ + float min = minVals[f][layer-1]; + float max = maxVals[f][layer-1]; + features[f] = (features[f] - min) / (max-min); + } + features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); + + List sectionList = getSectionList(layer, sector); + for(int section : sectionList){ + for(int f = 0; f < NFEATURES; f++){ + x[section - 1][nHits[section - 1]][f] = features[f]; + } + + mask[section - 1][nHits[section - 1]] = 1.0f; + + maps[section - 1].put(nHits[section - 1], hit); + + nHits[section - 1]++; + if(nHits[section - 1] == MAX_HITS) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); + return true; + } + } } // Read BMT bank and set input for models @@ -287,64 +283,64 @@ public boolean processDataEvent(DataEvent event) { for(int i = 0; i < bmt_bank.rows(); i++){ int sector = bmt_bank.getByte("sector", i); int layer = bmt_bank.getByte("layer", i); - int strip = bmt_bank.getShort("component",i); - int order = bmt_bank.getByte("order", i); - int adc = bst_bank.getInt("ADC", i); + int strip = bmt_bank.getShort("strip",i); Hit hit = new Hit(layer+6, sector, strip, nHitsLayerSectorStrip_BMT[layer-1][sector-1][strip-1]++); - if(order == 0 || order == 10){ - Point3D originPoint, endPoint; - int region = (layer + 1) / 2; // Get region number for a BMT layer; layers 1, 4, 6 for BMT-C layers, and layers 2, 3, 5 for BMT-Z - if(layer == 2 || layer ==3 || layer == 5) { - Line3D line = Geometry.getInstance().getBMT().getLCZstrip(region, sector, strip, swimmer); - originPoint = line.origin(); - endPoint = line.origin(); - } - else { - Arc3D arcLine = Geometry.getInstance().getBMT().getCstrip(region, sector, strip); - originPoint = arcLine.origin(); - endPoint = arcLine.end(); - } - - layer += 6; - float[] features = {strip, (float)(originPoint.x()/10.), (float)(endPoint.x()/10.), (float)(originPoint.y()/10.), - (float)(endPoint.y()/10.), (float)(originPoint.z()/10.), (float)(endPoint.z()/10.), sector, layer}; // unit conversion from mm to cm for end points - - for(int f = 0; f < NFEATURES - 1; f++){ - float min = minVals[f][layer-1]; - float max = maxVals[f][layer-1]; - features[f] = (features[f] - min) / (max-min); - } - features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); - - List sectionList = getSectionList(layer, sector); - for(int section : sectionList){ - for(int f = 0; f < NFEATURES; f++){ - x[section - 1][nHits[section - 1]][f] = features[f]; - } - - mask[section - 1][nHits[section - 1]] = 1.0f; - - maps[section - 1].put(nHits[section - 1], hit); - - nHits[section - 1]++; - if(nHits[section - 1] == MAX_HITS) { - Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); - return true; - } - } + Point3D originPoint, endPoint; + int region = (layer + 1) / 2; // Get region number for a BMT layer; layers 1, 4, 6 for BMT-C layers, and layers 2, 3, 5 for BMT-Z + if(layer == 2 || layer ==3 || layer == 5) { + Line3D line = Geometry.getInstance().getBMT().getLCZstrip(region, sector, strip, swimmer); + originPoint = line.origin(); + endPoint = line.origin(); } + else { + Arc3D arcLine = Geometry.getInstance().getBMT().getCstrip(region, sector, strip); + originPoint = arcLine.origin(); + endPoint = arcLine.end(); + } + + layer += 6; + float[] features = {strip, (float)(originPoint.x()/10.), (float)(endPoint.x()/10.), (float)(originPoint.y()/10.), + (float)(endPoint.y()/10.), (float)(originPoint.z()/10.), (float)(endPoint.z()/10.), sector, layer}; // unit conversion from mm to cm for end points + + for(int f = 0; f < NFEATURES - 1; f++){ + float min = minVals[f][layer-1]; + float max = maxVals[f][layer-1]; + features[f] = (features[f] - min) / (max-min); + } + features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); + + List sectionList = getSectionList(layer, sector); + for(int section : sectionList){ + for(int f = 0; f < NFEATURES; f++){ + x[section - 1][nHits[section - 1]][f] = features[f]; + } + + mask[section - 1][nHits[section - 1]] = 1.0f; + + maps[section - 1].put(nHits[section - 1], hit); + + nHits[section - 1]++; + if(nHits[section - 1] == MAX_HITS) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); + return true; + } + } } + + // AI-based hit classification. + // The status is encoded in a byte (3-bit bitmask). + // + // Each bit corresponds to one section: + // Section 1 <-> bit 0 + // Section 2 <-> bit 1 + // Section 3 <-> bit 2 + // + // Bit value: + // 0 = noise + // 1 = signal - // Apply models for prediction of hits - // Status for a hit with order of 0: - // 0: rejected - // 1: accepted by section 1 - // 2: accepted by section 2 - // 3: accepted by section 3 - // 12: accepted by sections 1 & 2; Note: a hit is shared by secton 1 & 2, and is accpcted by both section - // 23: accepted by sections 2 & 3; Note: a hit is shared by secton 2 & 3, and is accpcted by both section int maxIndex_BST = Integer.MIN_VALUE; for (int i = 0; i < nHitsLayerSectorStrip_BST.length; i++) { for (int j = 0; j < nHitsLayerSectorStrip_BST[i].length; j++) { @@ -377,10 +373,7 @@ public boolean processDataEvent(DataEvent event) { float[][] preds = predictor.predict(input); for(int i = 0; i < nHits[s]; i++){ Hit hit = maps[s].get(i); - byte status = statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()]; - if(preds[i][0] > thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] = (byte) (status * 10 + s + 1); - //if(statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] < 0 || statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] > 123) - // System.out.println(preds[i][0] + " " + s + " " + i + " " + hit.getLayer() + " " + hit.getSector() + " " + hit.getStrip() + " " + hit.getIndex() + " " + statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()]); + if(preds[i][0] > thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << s); } } finally { predictors[s].put(predictor); @@ -482,28 +475,24 @@ private void updateBanks(DataBank bst_bank, DataBank bmt_bank, byte[][][][] stat for (int row=0; row Date: Tue, 17 Mar 2026 15:32:34 -0400 Subject: [PATCH 04/14] update byte for record of hit status by AI prediction --- etc/bankdefs/hipo4/bmt.json | 2 +- etc/bankdefs/hipo4/bst.json | 2 +- .../rec/cvt/services/CVTDenoiseEngine.java | 23 +++++++------------ 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/etc/bankdefs/hipo4/bmt.json b/etc/bankdefs/hipo4/bmt.json index 3d50008262..d46630c226 100644 --- a/etc/bankdefs/hipo4/bmt.json +++ b/etc/bankdefs/hipo4/bmt.json @@ -130,7 +130,7 @@ {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, {"name":"status", "type":"B", "info":"status (0=good)"}, - {"name":"ai", "type":"B", "info":"3 bits corresponds to 3 sections; 1 = singal and 0 = noise"} + {"name":"ai", "type":"B", "info":"bits [0-2] : noise flags for sections 1-3 (1 = noise, 0 = signal); bits [3-5] : hit presence for sections 1-3 (1 = hit present, 0 = no hit)"} ] }, { diff --git a/etc/bankdefs/hipo4/bst.json b/etc/bankdefs/hipo4/bst.json index a45c3475b1..7b8acb882c 100644 --- a/etc/bankdefs/hipo4/bst.json +++ b/etc/bankdefs/hipo4/bst.json @@ -121,7 +121,7 @@ {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, {"name":"status", "type":"B", "info":"status (0=good)"}, - {"name":"ai", "type":"B", "info":"3 bits corresponds to 3 sections; 1 = singal and 0 = noise"} + {"name":"ai", "type":"B", "info":"bits [0-2] : noise flags for sections 1-3 (1 = noise, 0 = signal); bits [3-5] : hit presence for sections 1-3 (1 = hit present, 0 = no hit)"} ] }, { diff --git a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java index f860056984..29030ca589 100644 --- a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java +++ b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java @@ -139,7 +139,7 @@ public class CVTDenoiseEngine extends ReconstructionEngine { private static final int MAX_HITS = 450; // Maximum of hits for each layer private static final int NFEATURES = 9; // Number of features for input of models - // Inputs are from BST and BMT adc banks + // Inputs are from BST and BMT hit banks final static String BST_BANK = "BST::Hits"; final static String BMT_BANK = "BMT::Hits"; @@ -328,19 +328,7 @@ public boolean processDataEvent(DataEvent event) { } } } - - // AI-based hit classification. - // The status is encoded in a byte (3-bit bitmask). - // - // Each bit corresponds to one section: - // Section 1 <-> bit 0 - // Section 2 <-> bit 1 - // Section 3 <-> bit 2 - // - // Bit value: - // 0 = noise - // 1 = signal - + int maxIndex_BST = Integer.MIN_VALUE; for (int i = 0; i < nHitsLayerSectorStrip_BST.length; i++) { for (int j = 0; j < nHitsLayerSectorStrip_BST[i].length; j++) { @@ -363,6 +351,10 @@ public boolean processDataEvent(DataEvent event) { } int maxIndex = Math.max(maxIndex_BST, maxIndex_BMT); + // AI-based hit classification. + // Hit status is encoded in a single byte: + // bits [0–2] : noise flags for sections 1–3 (1 = noise, 0 = signal) + // bits [3–5] : hit presence for sections 1–3 (1 = hit present, 0 = no hit) byte[][][][] statuses = new byte[NLAYERS][18][1152][maxIndex+1]; for(int s = 0; s < NSECTIONS; s++){ CVTInput input = new CVTInput(x[s],mask[s]); @@ -373,7 +365,8 @@ public boolean processDataEvent(DataEvent event) { float[][] preds = predictor.predict(input); for(int i = 0; i < nHits[s]; i++){ Hit hit = maps[s].get(i); - if(preds[i][0] > thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << s); + statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << (s+3)); + if(preds[i][0] < thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << s); } } finally { predictors[s].put(predictor); From 49a2701a74414eb4925960ac868100eac99b5f2e Mon Sep 17 00:00:00 2001 From: veronique Date: Thu, 23 Apr 2026 13:51:34 -0400 Subject: [PATCH 05/14] hipo to npz converter python script to view to converted output --- .../main/java/org/jlab/io/hipo/HipoToNpz.java | 966 ++++++++++++++++++ 1 file changed, 966 insertions(+) create mode 100644 common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java new file mode 100644 index 0000000000..a77b1364df --- /dev/null +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java @@ -0,0 +1,966 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package org.jlab.io.hipo; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.zip.CRC32; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.jlab.io.base.DataBank; +import org.jlab.io.base.DataEvent; +/** + * + * @author veronique + */ + +/** + * Convert selected banks from a HIPO file into a single NPZ archive. + * + * Bank selection: + * - no bank selection specified: include all banks found in the HIPO file + * - comma-separated bank names: include only those banks + * - --bank-file : one bank name per line, '#' comments allowed + * - both comma list and --bank-file may be used together + * + * Schema JSON selection: + * - --schema-dir is required + * - if banks are selected, only JSON files containing at least one selected bank are parsed + * - if no banks are selected, all JSON files in the folder are parsed + * + * Example: after compiling coatjava, cd coatjava, run the script: + * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json + * + * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json BST::adc,RUN::config + * + * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json --bank-file banks.txt + */ +public class HipoToNpz { + + public static void main(String[] args) throws Exception { + CliOptions options = CliOptions.parse(args); + + Map schemaIndex = SchemaLoader.indexSchemaFiles(options.schemaDir); + Set schemaFilesToLoad = SchemaLoader.chooseSchemaFiles(schemaIndex, options.selectedBanks); + + if (schemaFilesToLoad.isEmpty()) { + throw new IllegalStateException("No schema JSON files selected from " + options.schemaDir.getAbsolutePath()); + } + + Map schemaTypes = SchemaLoader.loadSchemaTypes(schemaFilesToLoad); + + System.out.println("Schema dir : " + options.schemaDir.getAbsolutePath()); + System.out.println("Schema files : " + schemaFilesToLoad.size()); + if (options.selectedBanks == null) { + System.out.println("Selected banks : ALL"); + } else { + System.out.println("Selected banks : " + options.selectedBanks.size()); + for (String b : options.selectedBanks) { + System.out.println(" " + b); + } + } + + Converter converter = new Converter(options.selectedBanks, schemaTypes); + converter.convert(options.input, options.output); + } + + // ------------------------------------------------------------------------ + // CLI + // ------------------------------------------------------------------------ + + private static final class CliOptions { + final File input; + final File output; + final File schemaDir; + final Set selectedBanks; // null means all banks + + private CliOptions(File input, File output, File schemaDir, Set selectedBanks) { + this.input = input; + this.output = output; + this.schemaDir = schemaDir; + this.selectedBanks = selectedBanks; + } + + static CliOptions parse(String[] args) throws Exception { + if (args.length < 4) { + printUsageAndExit(); + } + + File input = new File(args[0]); + File output = new File(args[1]); + + File schemaDir = null; + Set selectedBanks = new LinkedHashSet<>(); + boolean selectAll = true; + + for (int i = 2; i < args.length; i++) { + String arg = args[i]; + if (arg == null || arg.isBlank()) { + continue; + } + + if ("--schema-dir".equals(arg)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--schema-dir requires a directory path"); + } + schemaDir = new File(args[++i]); + continue; + } + + if ("--bank-file".equals(arg)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--bank-file requires a file path"); + } + File bankFile = new File(args[++i]); + selectedBanks.addAll(readBankNames(bankFile)); + selectAll = false; + continue; + } + + if ("*".equals(arg)) { + selectedBanks.clear(); + selectAll = true; + continue; + } + + for (String s : arg.split(",")) { + String bank = s.trim(); + if (!bank.isEmpty()) { + selectedBanks.add(bank); + selectAll = false; + } + } + } + + if (schemaDir == null) { + throw new IllegalArgumentException("--schema-dir is required"); + } + if (!schemaDir.isDirectory()) { + throw new IllegalArgumentException("Schema directory not found: " + schemaDir.getAbsolutePath()); + } + if (!input.exists()) { + throw new IllegalArgumentException("Input HIPO file not found: " + input.getAbsolutePath()); + } + + return new CliOptions(input, output, schemaDir, selectAll ? null : selectedBanks); + } + + private static Set readBankNames(File bankFile) throws IOException { + if (!bankFile.exists()) { + throw new IOException("Bank file not found: " + bankFile.getAbsolutePath()); + } + + Set names = new LinkedHashSet<>(); + for (String line : Files.readAllLines(bankFile.toPath(), StandardCharsets.UTF_8)) { + String s = line.trim(); + if (s.isEmpty() || s.startsWith("#")) { + continue; + } + names.add(s); + } + return names; + } + + private static void printUsageAndExit() { + System.err.println("Usage:"); + System.err.println(" java org.jlab.io.hipo.HipoToNpz --schema-dir [bank1,bank2,...|*] [--bank-file banks.txt]"); + System.exit(2); + } + } + + // ------------------------------------------------------------------------ + // Types + // ------------------------------------------------------------------------ + + private enum ColumnType { + BYTE(" BYTE; + case "S" -> SHORT; + case "I" -> INT; + case "L" -> LONG; + case "F" -> FLOAT; + case "D" -> DOUBLE; + default -> throw new IllegalStateException("Unknown schema type '" + code + "' for " + fullName); + }; + } + } + + // ------------------------------------------------------------------------ + // Converter + // ------------------------------------------------------------------------ + + private static final class Converter { + private final Map banks = new LinkedHashMap<>(); + private final Set selectedBanks; // null means all banks + private final Map schemaTypes; // BANK/COLUMN -> type + + Converter(Set selectedBanks, Map schemaTypes) { + this.selectedBanks = selectedBanks; + this.schemaTypes = schemaTypes; + } + + void convert(File input, File output) throws Exception { + if (selectedBanks == null) { + System.out.println("Including all banks"); + } else { + System.out.println("Including selected banks only"); + } + + HipoDataSource reader = new HipoDataSource(); + reader.open(input); + + long nEvents = 0; + while (reader.hasEvent()) { + DataEvent event = reader.getNextEvent(); + nEvents++; + ingestEvent(event); + + if ((nEvents % 10000) == 0) { + System.out.printf("Processed %,d events%n", nEvents); + } + } + reader.close(); + + writeNpz(output); + System.out.printf("Wrote %s with %,d events and %,d banks%n", + output.getAbsolutePath(), nEvents, banks.size()); + } + + private boolean keepBank(String bankName) { + return selectedBanks == null || selectedBanks.contains(bankName); + } + + private void ingestEvent(DataEvent event) { + String[] bankNames = event.getBankList(); + if (bankNames == null) { + return; + } + + Map presentRows = new HashMap<>(); + + for (String bankName : bankNames) { + if (bankName == null || bankName.isBlank()) { + continue; + } + if (!keepBank(bankName)) { + continue; + } + + DataBank bank = event.getBank(bankName); + if (bank == null) { + continue; + } + + int rows = bank.rows(); + presentRows.put(bankName, rows); + + BankStore store = banks.computeIfAbsent(bankName, BankStore::new); + ensureColumns(store, bank); + + String[] cols = bank.getColumnList(); + if (cols != null) { + for (String col : cols) { + ColumnStore cstore = store.columns.get(col); + if (cstore == null) { + continue; + } + for (int r = 0; r < rows; r++) { + cstore.append(bank, col, r); + } + } + } + } + + for (BankStore store : banks.values()) { + int rows = presentRows.getOrDefault(store.bankName, 0); + store.appendEventRows(rows); + } + } + + private void ensureColumns(BankStore store, DataBank bank) { + String[] cols = bank.getColumnList(); + if (cols == null) { + return; + } + + for (String col : cols) { + if (store.columns.containsKey(col)) { + continue; + } + ColumnType type = discoverColumnType(bank, col); + store.columns.put(col, new ColumnStore(store.bankName, col, type)); + } + } + + private ColumnType discoverColumnType(DataBank bank, String col) { + String fullName = bank.getDescriptor().getName() + "/" + col; + + ColumnType schemaType = schemaTypes.get(fullName); + if (schemaType != null) { + return schemaType; + } + + // Fallback to descriptor only if schema map lacks the bank/column. + Object desc = bank.getDescriptor(); + if (desc == null) { + throw new IllegalStateException("No descriptor and no schema entry for " + fullName); + } + + String typeName = tryGetTypeName(desc, col); + if (typeName != null) { + String s = typeName.trim(); + if (!s.isEmpty() && !s.equalsIgnoreCase("undefined")) { + return mapTypeName(s, fullName); + } + } + + Integer typeCode = tryGetTypeCode(desc, col); + if (typeCode != null) { + return mapTypeCode(typeCode, fullName); + } + + throw new IllegalStateException("Could not determine type for " + fullName); + } + + private Integer tryGetTypeCode(Object desc, String col) { + try { + Object v = desc.getClass() + .getMethod("getProperty", String.class, String.class) + .invoke(desc, "type", col); + if (v instanceof Number n) { + return n.intValue(); + } + } catch (ReflectiveOperationException ignored) { + } + + try { + Object v = desc.getClass() + .getMethod("getProperty", String.class) + .invoke(desc, col + ".type"); + if (v instanceof Number n) { + return n.intValue(); + } + } catch (ReflectiveOperationException ignored) { + } + + return null; + } + + private String tryGetTypeName(Object desc, String col) { + try { + Object v = desc.getClass() + .getMethod("getPropertyString", String.class, String.class) + .invoke(desc, "type", col); + if (v != null) { + return v.toString(); + } + } catch (ReflectiveOperationException ignored) { + } + + try { + Object v = desc.getClass() + .getMethod("getPropertyString", String.class) + .invoke(desc, col + ".type"); + if (v != null) { + return v.toString(); + } + } catch (ReflectiveOperationException ignored) { + } + + return null; + } + + private ColumnType mapTypeCode(int t, String fullName) { + return switch (t) { + case 1 -> ColumnType.INT; + case 2 -> ColumnType.FLOAT; + case 3 -> ColumnType.DOUBLE; + case 4 -> ColumnType.SHORT; + case 5 -> ColumnType.BYTE; + case 6 -> ColumnType.LONG; + case 8 -> ColumnType.BYTE; + default -> throw new IllegalStateException("Unknown type code " + t + " for " + fullName); + }; + } + + private ColumnType mapTypeName(String typeName, String fullName) { + String s = typeName.trim().toLowerCase(); + return switch (s) { + case "int", "int32", "i4" -> ColumnType.INT; + case "float", "float32", "f4" -> ColumnType.FLOAT; + case "double", "float64", "f8" -> ColumnType.DOUBLE; + case "short", "int16", "i2" -> ColumnType.SHORT; + case "byte", "int8", "i1" -> ColumnType.BYTE; + case "long", "int64", "i8" -> ColumnType.LONG; + default -> throw new IllegalStateException("Unknown type name '" + typeName + "' for " + fullName); + }; + } + + private void writeNpz(File output) throws IOException { + try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(output)))) { + zos.setLevel(Deflater.BEST_SPEED); + + for (BankStore bank : banks.values()) { + String bankBase = sanitize(bank.bankName); + + addEntry(zos, bankBase + "__rows_per_event.npy", + Npy.writeIntArray(bank.rowsPerEvent.toArray(), " indexSchemaFiles(File schemaDir) throws IOException { + File[] files = schemaDir.listFiles((dir, name) -> name.toLowerCase().endsWith(".json")); + if (files == null || files.length == 0) { + throw new IOException("No .json schema files found in " + schemaDir.getAbsolutePath()); + } + + Map bankToFile = new HashMap<>(); + for (File file : files) { + Object root = Json.parse(Files.readString(file.toPath(), StandardCharsets.UTF_8)); + if (!(root instanceof List banks)) { + continue; + } + + for (Object bankObj : banks) { + if (!(bankObj instanceof Map bankMap)) { + continue; + } + Object nameObj = bankMap.get("name"); + if (nameObj instanceof String bankName && !bankName.isBlank()) { + bankToFile.put(bankName, file); + } + } + } + return bankToFile; + } + + static Set chooseSchemaFiles(Map bankToFile, Set selectedBanks) { + Set files = new LinkedHashSet<>(); + + if (selectedBanks == null) { + files.addAll(bankToFile.values()); + return files; + } + + List missing = new ArrayList<>(); + for (String bank : selectedBanks) { + File file = bankToFile.get(bank); + if (file != null) { + files.add(file); + } else { + missing.add(bank); + } + } + + if (!missing.isEmpty()) { + System.err.println("Warning: no schema JSON file found for selected banks:"); + for (String b : missing) { + System.err.println(" " + b); + } + } + + return files; + } + + static Map loadSchemaTypes(Set schemaFiles) throws IOException { + Map types = new HashMap<>(); + + List sorted = new ArrayList<>(schemaFiles); + sorted.sort(Comparator.comparing(File::getName)); + + for (File file : sorted) { + Object root = Json.parse(Files.readString(file.toPath(), StandardCharsets.UTF_8)); + if (!(root instanceof List banks)) { + continue; + } + + for (Object bankObj : banks) { + if (!(bankObj instanceof Map bankMap)) { + continue; + } + + Object nameObj = bankMap.get("name"); + Object entriesObj = bankMap.get("entries"); + if (!(nameObj instanceof String bankName) || !(entriesObj instanceof List entries)) { + continue; + } + + for (Object entryObj : entries) { + if (!(entryObj instanceof Map entryMap)) { + continue; + } + + Object entryNameObj = entryMap.get("name"); + Object entryTypeObj = entryMap.get("type"); + if (!(entryNameObj instanceof String colName) || !(entryTypeObj instanceof String typeCode)) { + continue; + } + + String fullName = bankName + "/" + colName; + types.put(fullName, ColumnType.fromSchemaCode(typeCode, fullName)); + } + } + } + + return types; + } + } + + // ------------------------------------------------------------------------ + // Stores + // ------------------------------------------------------------------------ + + private static final class BankStore { + final String bankName; + final Map columns = new LinkedHashMap<>(); + final IntList rowsPerEvent = new IntList(); + final LongList offsets = new LongList(); + long totalRows = 0; + + BankStore(String bankName) { + this.bankName = bankName; + this.offsets.add(0L); + } + + void appendEventRows(int rows) { + rowsPerEvent.add(rows); + totalRows += rows; + offsets.add(totalRows); + } + } + + private static final class ColumnStore { + final String bankName; + final String columnName; + final ColumnType type; + final ByteList bytes; + final ShortList shorts; + final IntList ints; + final LongList longs; + final FloatList floats; + final DoubleList doubles; + + ColumnStore(String bankName, String columnName, ColumnType type) { + this.bankName = bankName; + this.columnName = columnName; + this.type = type; + this.bytes = type == ColumnType.BYTE ? new ByteList() : null; + this.shorts = type == ColumnType.SHORT ? new ShortList() : null; + this.ints = type == ColumnType.INT ? new IntList() : null; + this.longs = type == ColumnType.LONG ? new LongList() : null; + this.floats = type == ColumnType.FLOAT ? new FloatList() : null; + this.doubles = type == ColumnType.DOUBLE ? new DoubleList() : null; + } + + void append(DataBank bank, String col, int row) { + switch (type) { + case BYTE -> bytes.add(bank.getByte(col, row)); + case SHORT -> shorts.add(bank.getShort(col, row)); + case INT -> ints.add(bank.getInt(col, row)); + case LONG -> longs.add(bank.getLong(col, row)); + case FLOAT -> floats.add(bank.getFloat(col, row)); + case DOUBLE -> doubles.add(bank.getDouble(col, row)); + } + } + + byte[] toNpyBytes() throws IOException { + return switch (type) { + case BYTE -> Npy.writeByteArray(bytes.toArray(), type.npyDescr); + case SHORT -> Npy.writeShortArray(shorts.toArray(), type.npyDescr); + case INT -> Npy.writeIntArray(ints.toArray(), type.npyDescr); + case LONG -> Npy.writeLongArray(longs.toArray(), type.npyDescr); + case FLOAT -> Npy.writeFloatArray(floats.toArray(), type.npyDescr); + case DOUBLE -> Npy.writeDoubleArray(doubles.toArray(), type.npyDescr); + }; + } + } + + // ------------------------------------------------------------------------ + // Minimal JSON parser + // ------------------------------------------------------------------------ + + private static final class Json { + static Object parse(String text) { + return new Parser(text).parseValue(); + } + + private static final class Parser { + private final String s; + private int i = 0; + + Parser(String s) { + this.s = s; + } + + Object parseValue() { + skipWs(); + if (i >= s.length()) { + throw new IllegalStateException("Unexpected end of JSON"); + } + + char c = s.charAt(i); + return switch (c) { + case '{' -> parseObject(); + case '[' -> parseArray(); + case '"' -> parseString(); + case 't' -> parseTrue(); + case 'f' -> parseFalse(); + case 'n' -> parseNull(); + default -> parseNumber(); + }; + } + + private Map parseObject() { + expect('{'); + Map obj = new LinkedHashMap<>(); + skipWs(); + if (peek('}')) { + expect('}'); + return obj; + } + + while (true) { + skipWs(); + String key = parseString(); + skipWs(); + expect(':'); + Object value = parseValue(); + obj.put(key, value); + skipWs(); + if (peek('}')) { + expect('}'); + break; + } + expect(','); + } + return obj; + } + + private List parseArray() { + expect('['); + List arr = new ArrayList<>(); + skipWs(); + if (peek(']')) { + expect(']'); + return arr; + } + + while (true) { + arr.add(parseValue()); + skipWs(); + if (peek(']')) { + expect(']'); + break; + } + expect(','); + } + return arr; + } + + private String parseString() { + expect('"'); + StringBuilder sb = new StringBuilder(); + while (i < s.length()) { + char c = s.charAt(i++); + if (c == '"') { + return sb.toString(); + } + if (c == '\\') { + if (i >= s.length()) { + throw new IllegalStateException("Bad escape"); + } + char e = s.charAt(i++); + switch (e) { + case '"', '\\', '/' -> sb.append(e); + case 'b' -> sb.append('\b'); + case 'f' -> sb.append('\f'); + case 'n' -> sb.append('\n'); + case 'r' -> sb.append('\r'); + case 't' -> sb.append('\t'); + case 'u' -> { + if (i + 4 > s.length()) { + throw new IllegalStateException("Bad unicode escape"); + } + String hex = s.substring(i, i + 4); + sb.append((char) Integer.parseInt(hex, 16)); + i += 4; + } + default -> throw new IllegalStateException("Bad escape: \\" + e); + } + } else { + sb.append(c); + } + } + throw new IllegalStateException("Unterminated string"); + } + + private Object parseNumber() { + int start = i; + if (s.charAt(i) == '-') { + i++; + } + while (i < s.length() && Character.isDigit(s.charAt(i))) { + i++; + } + boolean isFloat = false; + if (i < s.length() && s.charAt(i) == '.') { + isFloat = true; + i++; + while (i < s.length() && Character.isDigit(s.charAt(i))) { + i++; + } + } + if (i < s.length() && (s.charAt(i) == 'e' || s.charAt(i) == 'E')) { + isFloat = true; + i++; + if (i < s.length() && (s.charAt(i) == '+' || s.charAt(i) == '-')) { + i++; + } + while (i < s.length() && Character.isDigit(s.charAt(i))) { + i++; + } + } + + String num = s.substring(start, i); + return isFloat ? Double.parseDouble(num) : Long.parseLong(num); + } + + private Boolean parseTrue() { + expect('t'); expect('r'); expect('u'); expect('e'); + return Boolean.TRUE; + } + + private Boolean parseFalse() { + expect('f'); expect('a'); expect('l'); expect('s'); expect('e'); + return Boolean.FALSE; + } + + private Object parseNull() { + expect('n'); expect('u'); expect('l'); expect('l'); + return null; + } + + private void skipWs() { + while (i < s.length()) { + char c = s.charAt(i); + if (c == ' ' || c == '\n' || c == '\r' || c == '\t') { + i++; + } else { + break; + } + } + } + + private boolean peek(char c) { + skipWs(); + return i < s.length() && s.charAt(i) == c; + } + + private void expect(char c) { + skipWs(); + if (i >= s.length() || s.charAt(i) != c) { + throw new IllegalStateException("Expected '" + c + "' at position " + i); + } + i++; + } + } + } + + // ------------------------------------------------------------------------ + // NPY writer + // ------------------------------------------------------------------------ + + private static final class Npy { + private static final byte[] MAGIC = {(byte) 0x93, 'N', 'U', 'M', 'P', 'Y'}; + + static byte[] writeByteArray(byte[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + out.write(values); + return out.toByteArray(); + } + + static byte[] writeShortArray(short[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 2).order(ByteOrder.LITTLE_ENDIAN); + for (short v : values) bb.putShort(v); + out.write(bb.array()); + return out.toByteArray(); + } + + static byte[] writeIntArray(int[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); + for (int v : values) bb.putInt(v); + out.write(bb.array()); + return out.toByteArray(); + } + + static byte[] writeLongArray(long[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); + for (long v : values) bb.putLong(v); + out.write(bb.array()); + return out.toByteArray(); + } + + static byte[] writeFloatArray(float[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); + for (float v : values) bb.putFloat(v); + out.write(bb.array()); + return out.toByteArray(); + } + + static byte[] writeDoubleArray(double[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); + for (double v : values) bb.putDouble(v); + out.write(bb.array()); + return out.toByteArray(); + } + + private static void writeHeader(ByteArrayOutputStream out, String descr, int length) throws IOException { + out.write(MAGIC); + out.write(1); + out.write(0); + + String dict = "{'descr': '" + descr + "', 'fortran_order': False, 'shape': (" + length + ",), }"; + int preamble = MAGIC.length + 2 + 2; + int padLen = 16 - ((preamble + dict.length() + 1) % 16); + if (padLen == 16) { + padLen = 0; + } + + String fullHeader = dict + " ".repeat(padLen) + "\n"; + byte[] fullHeaderBytes = fullHeader.getBytes(StandardCharsets.US_ASCII); + + ByteBuffer hlen = ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN); + hlen.putShort((short) fullHeaderBytes.length); + out.write(hlen.array()); + out.write(fullHeaderBytes); + } + } + + // ------------------------------------------------------------------------ + // Primitive dynamic arrays + // ------------------------------------------------------------------------ + + private static final class ByteList { + private byte[] data = new byte[1024]; + private int size = 0; + void add(byte v) { ensure(size + 1); data[size++] = v; } + byte[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class ShortList { + private short[] data = new short[1024]; + private int size = 0; + void add(short v) { ensure(size + 1); data[size++] = v; } + short[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class IntList { + private int[] data = new int[1024]; + private int size = 0; + void add(int v) { ensure(size + 1); data[size++] = v; } + int[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class LongList { + private long[] data = new long[1024]; + private int size = 0; + void add(long v) { ensure(size + 1); data[size++] = v; } + long[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class FloatList { + private float[] data = new float[1024]; + private int size = 0; + void add(float v) { ensure(size + 1); data[size++] = v; } + float[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class DoubleList { + private double[] data = new double[1024]; + private int size = 0; + void add(double v) { ensure(size + 1); data[size++] = v; } + double[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } +} \ No newline at end of file From ff9a3f97e64f08f3ab13edc7b8a7bae6ca8341e5 Mon Sep 17 00:00:00 2001 From: veronique Date: Thu, 23 Apr 2026 14:02:13 -0400 Subject: [PATCH 06/14] hipo to npz converter python script to view to converted output --- HipoToNpz_README | 10 +++ bin/hipoToNpz | 10 +++ list_npz_banks.py | 166 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 HipoToNpz_README create mode 100755 bin/hipoToNpz create mode 100644 list_npz_banks.py diff --git a/HipoToNpz_README b/HipoToNpz_README new file mode 100644 index 0000000000..8f6ac0517f --- /dev/null +++ b/HipoToNpz_README @@ -0,0 +1,10 @@ +To convert a hipo file to npz format for a given set of banks: after compiling coatjava, cd coatjava, run the script: + * for all available banks: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 + * for BST::adc,RUN::config banks: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 BST::adc,RUN::config + * for banks listed in banks.txt: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 --bank-file banks.txt + +To inspect the npz file for n events: + * python3 list_npz_banks.py /pathTo/MyFile.npz n +Or + * python list_npz_banks.py /pathTo/MyFile.npz n + diff --git a/bin/hipoToNpz b/bin/hipoToNpz new file mode 100755 index 0000000000..da090d90af --- /dev/null +++ b/bin/hipoToNpz @@ -0,0 +1,10 @@ +#!/bin/bash + +. `dirname $0`/../libexec/env.sh + +export MALLOC_ARENA_MAX=1 + +java ${JAVA_OPTS-} -Xmx1536m -Xms1024m -XX:+UseSerialGC \ + -cp ${COATJAVA_CLASSPATH:-''} \ + org.jlab.io.hipo.HipoToNpz \ + $* diff --git a/list_npz_banks.py b/list_npz_banks.py new file mode 100644 index 0000000000..930e930a9d --- /dev/null +++ b/list_npz_banks.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +Show values for a few events from all banks in an NPZ produced by HipoToNpz. + +Usage: + python list_npz_banks.py file.npz + python list_npz_banks.py file.npz 3 +""" + +from __future__ import annotations + +import ast +import struct +import sys +from collections import defaultdict +from pathlib import Path +from zipfile import ZipFile + + +def read_npy_from_bytes(blob: bytes): + if len(blob) < 10 or blob[:6] != b"\x93NUMPY": + raise ValueError("Not a valid .npy payload") + + major = blob[6] + minor = blob[7] + + if major == 1: + header_len = struct.unpack(" 3: + print("Usage: python3 show_all_npz_events_no_numpy.py [N_EVENTS]") + sys.exit(2) + + npz_path = Path(sys.argv[1]) + n_events = int(sys.argv[2]) if len(sys.argv) == 3 else 3 + + if not npz_path.exists(): + print(f"File not found: {npz_path}") + sys.exit(1) + + arrays = load_npz(npz_path) + banks = collect_banks(arrays.keys()) + + for bank in banks: + rows_key = f"{bank}__rows_per_event" + offsets_key = f"{bank}__offsets" + + if rows_key not in arrays or offsets_key not in arrays: + continue + + rows_per_event = arrays[rows_key] + offsets = arrays[offsets_key] + + column_keys = sorted( + k for k in arrays.keys() + if k.startswith(bank + "__") and k not in (rows_key, offsets_key) + ) + + print("=" * 80) + print(f"BANK: {bank}") + print("COLUMNS:") + for key in column_keys: + print(f" {key[len(bank) + 2:]}") + print() + + nevt = min(n_events, len(rows_per_event)) + + for evt in range(nevt): + start = int(offsets[evt]) + stop = int(offsets[evt + 1]) + nrows = int(rows_per_event[evt]) + + print(f"Event {evt}: rows={nrows} slice=[{start}:{stop}]") + if nrows == 0: + print(" ") + print() + continue + + for key in column_keys: + col = key[len(bank) + 2:] + values = arrays[key][start:stop] + print(f" {col}: {list(values)}") + print() + + +if __name__ == "__main__": + main() From b62a37345802f5450592a72e9c07ec064c61ae14 Mon Sep 17 00:00:00 2001 From: veronique Date: Thu, 23 Apr 2026 14:02:59 -0400 Subject: [PATCH 07/14] yaml file to create FML bank --- etc/services/data-cvthits.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 etc/services/data-cvthits.yaml diff --git a/etc/services/data-cvthits.yaml b/etc/services/data-cvthits.yaml new file mode 100644 index 0000000000..90d59aaa45 --- /dev/null +++ b/etc/services/data-cvthits.yaml @@ -0,0 +1,30 @@ +io-services: + reader: + class: org.jlab.io.clara.HipoToHipoReader + name: HipoToHipoReader + writer: + class: org.jlab.io.clara.HipoToHipoWriter + name: HipoToHipoWriter +services: + - class: org.jlab.clas.swimtools.MagFieldsEngine + name: MAGFIELDS + - class: org.jlab.rec.cvt.services.CVTHitEngine + name: CVTH +configuration: + global: + variation: rga_fall2018_bg +# timestamp: 12/31/2020-00:00:00 +# triggerMask: "0x1" +## uncomment the following two lines for compatibility with alignments before the DC fixes +# dcMinistagger: "NOTONREFWIRE" +# dcFeedthroughs: "OFF" +# io-services: +# writer: +# schema_dir: dst + services: + MAGFIELDS: + magfieldSolenoidMap: Symm_solenoid_r601_phi1_z1201_13June2018.dat + magfieldTorusMap: Full_torus_r251_phi181_z251_25Jan2021.dat + +mime-types: + - binary/data-hipo From a1312331c9f6338a13ea1dea2d4b32e0f2eebb87 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 27 Jul 2026 16:10:01 -0400 Subject: [PATCH 08/14] refactor!: remove denoising stuff from this branch --- etc/bankdefs/hipo4/bmt.json | 3 +- etc/bankdefs/hipo4/bst.json | 3 +- etc/services/data-cvthits.yaml | 30 - reconstruction/cvt/pom.xml | 9 - .../rec/cvt/services/CVTDenoiseEngine.java | 515 ------------------ .../jlab/rec/cvt/services/CVTHitEngine.java | 447 --------------- 6 files changed, 2 insertions(+), 1005 deletions(-) delete mode 100644 etc/services/data-cvthits.yaml delete mode 100644 reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java delete mode 100644 reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTHitEngine.java diff --git a/etc/bankdefs/hipo4/bmt.json b/etc/bankdefs/hipo4/bmt.json index d46630c226..616793bd1f 100644 --- a/etc/bankdefs/hipo4/bmt.json +++ b/etc/bankdefs/hipo4/bmt.json @@ -129,8 +129,7 @@ {"name":"trkingStat", "type":"B", "info":"tracking status"}, {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, - {"name":"status", "type":"B", "info":"status (0=good)"}, - {"name":"ai", "type":"B", "info":"bits [0-2] : noise flags for sections 1-3 (1 = noise, 0 = signal); bits [3-5] : hit presence for sections 1-3 (1 = hit present, 0 = no hit)"} + {"name":"status", "type":"B", "info":"status (0=good)"} ] }, { diff --git a/etc/bankdefs/hipo4/bst.json b/etc/bankdefs/hipo4/bst.json index 7b8acb882c..8d9c6c46ba 100644 --- a/etc/bankdefs/hipo4/bst.json +++ b/etc/bankdefs/hipo4/bst.json @@ -120,8 +120,7 @@ {"name":"trkingStat", "type":"B", "info":"tracking status"}, {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, - {"name":"status", "type":"B", "info":"status (0=good)"}, - {"name":"ai", "type":"B", "info":"bits [0-2] : noise flags for sections 1-3 (1 = noise, 0 = signal); bits [3-5] : hit presence for sections 1-3 (1 = hit present, 0 = no hit)"} + {"name":"status", "type":"B", "info":"status (0=good)"} ] }, { diff --git a/etc/services/data-cvthits.yaml b/etc/services/data-cvthits.yaml deleted file mode 100644 index 90d59aaa45..0000000000 --- a/etc/services/data-cvthits.yaml +++ /dev/null @@ -1,30 +0,0 @@ -io-services: - reader: - class: org.jlab.io.clara.HipoToHipoReader - name: HipoToHipoReader - writer: - class: org.jlab.io.clara.HipoToHipoWriter - name: HipoToHipoWriter -services: - - class: org.jlab.clas.swimtools.MagFieldsEngine - name: MAGFIELDS - - class: org.jlab.rec.cvt.services.CVTHitEngine - name: CVTH -configuration: - global: - variation: rga_fall2018_bg -# timestamp: 12/31/2020-00:00:00 -# triggerMask: "0x1" -## uncomment the following two lines for compatibility with alignments before the DC fixes -# dcMinistagger: "NOTONREFWIRE" -# dcFeedthroughs: "OFF" -# io-services: -# writer: -# schema_dir: dst - services: - MAGFIELDS: - magfieldSolenoidMap: Symm_solenoid_r601_phi1_z1201_13June2018.dat - magfieldTorusMap: Full_torus_r251_phi181_z251_25Jan2021.dat - -mime-types: - - binary/data-hipo diff --git a/reconstruction/cvt/pom.xml b/reconstruction/cvt/pom.xml index 7a9db52551..50e7eecd3a 100644 --- a/reconstruction/cvt/pom.xml +++ b/reconstruction/cvt/pom.xml @@ -79,11 +79,6 @@ clas-geometry 14.1.2-SNAPSHOT - - org.jlab.clas12.detector - clas12detector-ai - 13.7.1-SNAPSHOT - org.jlab.clas12.detector clas12detector-eb @@ -100,10 +95,6 @@ junit junit - - ai.djl - api - diff --git a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java deleted file mode 100644 index 29030ca589..0000000000 --- a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java +++ /dev/null @@ -1,515 +0,0 @@ -package org.jlab.rec.cvt.services; - -import ai.djl.MalformedModelException; -import java.nio.file.Paths; -import ai.djl.ndarray.NDArray; -import ai.djl.ndarray.NDList; -import ai.djl.ndarray.NDManager; -import ai.djl.ndarray.types.Shape; -import ai.djl.repository.zoo.Criteria; -import ai.djl.repository.zoo.ZooModel; -import ai.djl.training.util.ProgressBar; -import ai.djl.translate.Translator; -import ai.djl.translate.TranslatorContext; -import ai.djl.translate.Batchifier; -import ai.djl.inference.Predictor; -import ai.djl.repository.zoo.ModelNotFoundException; -import ai.djl.translate.TranslateException; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.jlab.clas.reco.ReconstructionEngine; -import org.jlab.io.base.DataBank; -import org.jlab.io.base.DataEvent; -import org.jlab.utils.system.ClasUtilsFile; -import org.jlab.geom.prim.Line3D; -import org.jlab.geom.prim.Arc3D; -import org.jlab.geom.prim.Point3D; -import org.jlab.rec.cvt.Geometry; -import org.jlab.clas.swimtools.Swim; -import org.jlab.rec.cvt.Constants; -import org.jlab.service.ai.PredictorPool; -import org.jlab.utils.groups.IndexedTable; - -/* ----------------------------------------------- - Input class - ----------------------------------------------- */ -class CVTInput { - - float[][] data; // [450][9] - float[] mask; // [450] - - public CVTInput(float[][] data, float[] mask) { - this.data = data; - this.mask = mask; - } -} - -/* ----------------------------------------------- - DJL Translator - ----------------------------------------------- */ -class CVTTranslator implements Translator { - - @Override - public NDList processInput(TranslatorContext ctx, CVTInput input) { - - NDManager manager = ctx.getNDManager(); - - NDArray x = manager.create(input.data).reshape(1, 450, 9); - NDArray mask = manager.create(input.mask).reshape(1, 450); - - return new NDList(x, mask); - } - - @Override - public float[][] processOutput(TranslatorContext ctx, NDList list) { - - NDArray output = list.singletonOrThrow(); // shape: (1, 450, C) or (1, 450) - output = output.squeeze(0); // remove batch dim -> (450, C) - - float[] flat = output.toFloatArray(); - long[] shape = output.getShape().getShape(); - - int dim0 = (int) shape[0]; - int dim1 = shape.length > 1 ? (int) shape[1] : 1; - - float[][] result = new float[dim0][dim1]; - - for(int i=0;i[] criterias; - private ZooModel[] model; - private PredictorPool[] predictors; - - public CVTDenoiseEngine() { - super("CVTDenoiseEngine","Tongtong","1.0"); - } - - - @Override - public boolean init() { - this.initConstantsTables(); - - criterias = new Criteria[NSECTIONS]; - model = new ZooModel[NSECTIONS]; - predictors = new PredictorPool[NSECTIONS]; - - initScaling(); // Initial minVals and maxVals - - System.setProperty("ai.djl.pytorch.num_interop_threads", "1"); - System.setProperty("ai.djl.pytorch.num_threads", "1"); - System.setProperty("ai.djl.pytorch.graph_optimizer", "false"); - - - // Load models and set predictor pools - for(int i = 0; i < NSECTIONS; i++){ - if (getEngineConfigString(CONF_THRESHOLDS[i]) != null) - thresholds[i] = Float.parseFloat(getEngineConfigString(CONF_THRESHOLDS[i])); - if (getEngineConfigString(CONF_MODEL_FILES[i]) != null) - modelFiles[i] = getEngineConfigString(CONF_MODEL_FILES[i]); - - try { - String modelPath = ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/data/nnet/cvtdn/" + modelFiles[i]); - - CVTTranslator translator = new CVTTranslator(); - criterias[i] = Criteria.builder() - .setTypes(CVTInput.class, float[][].class) - .optModelPath(Paths.get(modelPath)) - .optEngine("PyTorch") - .optTranslator(translator) - .optProgress(new ProgressBar()) - .build(); - - model[i] = criterias[i].loadModel(); - - int threads = Integer.parseInt(getEngineConfigString(CONF_THREADS,"16")); - predictors[i] = new PredictorPool(threads, model[i]); - } catch (NullPointerException | MalformedModelException | IOException | ModelNotFoundException ex) { - Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, null, ex); - return false; - } - } - - return true; - } - - @Override - public boolean processDataEvent(DataEvent event) { - - Swim swimmer = new Swim(); - - int run = this.getRun(event); - - IndexedTable svtLorentz = this.getConstantsManager().getConstants(run, "/calibration/svt/lorentz_angle"); - IndexedTable bmtVoltage = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_voltage"); - - Geometry.getInstance().initialize(this.getConstantsManager().getVariation(), run, svtLorentz, bmtVoltage); - - if (!event.hasBank(BST_BANK)) return true; - if (!event.hasBank(BMT_BANK)) return true; - - DataBank bst_bank = event.getBank(BST_BANK); - DataBank bmt_bank = event.getBank(BMT_BANK); - - float[][][] x = new float[NSECTIONS][MAX_HITS][NFEATURES]; - float[][] mask = new float[NSECTIONS][MAX_HITS]; - - int[] nHits = {0, 0, 0}; - - Map[] maps = new HashMap[NSECTIONS]; - for (int i = 0; i < NSECTIONS; i++) { - maps[i] = new HashMap<>(); - } - - // Read BST bank and set input for models - int[][][] nHitsLayerSectorStrip_BST = new int[6][18][256]; // Number of hits in a strip - for(int i = 0; i < bst_bank.rows(); i++){ - int sector = bst_bank.getByte("sector", i); - int layer = bst_bank.getByte("layer", i); - int strip = bst_bank.getShort("strip",i); - - Hit hit = new Hit(layer, sector, strip, nHitsLayerSectorStrip_BST[layer-1][sector-1][strip-1]++); - - Line3D line = Geometry.getInstance().getSVT().getStrip(layer, sector, strip); - float[] features = {strip, (float)(line.origin().x()/10.), (float)(line.end().x()/10.), (float)(line.origin().y()/10.), - (float)(line.end().y()/10.), (float)(line.origin().z()/10.), (float)(line.end().z()/10.), sector, layer}; // unit conversion from mm to cm for end points - - for(int f = 0; f < NFEATURES - 1; f++){ - float min = minVals[f][layer-1]; - float max = maxVals[f][layer-1]; - features[f] = (features[f] - min) / (max-min); - } - features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); - - List sectionList = getSectionList(layer, sector); - for(int section : sectionList){ - for(int f = 0; f < NFEATURES; f++){ - x[section - 1][nHits[section - 1]][f] = features[f]; - } - - mask[section - 1][nHits[section - 1]] = 1.0f; - - maps[section - 1].put(nHits[section - 1], hit); - - nHits[section - 1]++; - if(nHits[section - 1] == MAX_HITS) { - Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); - return true; - } - } - } - - // Read BMT bank and set input for models - int[][][] nHitsLayerSectorStrip_BMT = new int[6][3][1152]; // Number of hits in a strip - for(int i = 0; i < bmt_bank.rows(); i++){ - int sector = bmt_bank.getByte("sector", i); - int layer = bmt_bank.getByte("layer", i); - int strip = bmt_bank.getShort("strip",i); - - Hit hit = new Hit(layer+6, sector, strip, nHitsLayerSectorStrip_BMT[layer-1][sector-1][strip-1]++); - - Point3D originPoint, endPoint; - int region = (layer + 1) / 2; // Get region number for a BMT layer; layers 1, 4, 6 for BMT-C layers, and layers 2, 3, 5 for BMT-Z - if(layer == 2 || layer ==3 || layer == 5) { - Line3D line = Geometry.getInstance().getBMT().getLCZstrip(region, sector, strip, swimmer); - originPoint = line.origin(); - endPoint = line.origin(); - } - else { - Arc3D arcLine = Geometry.getInstance().getBMT().getCstrip(region, sector, strip); - originPoint = arcLine.origin(); - endPoint = arcLine.end(); - } - - layer += 6; - float[] features = {strip, (float)(originPoint.x()/10.), (float)(endPoint.x()/10.), (float)(originPoint.y()/10.), - (float)(endPoint.y()/10.), (float)(originPoint.z()/10.), (float)(endPoint.z()/10.), sector, layer}; // unit conversion from mm to cm for end points - - for(int f = 0; f < NFEATURES - 1; f++){ - float min = minVals[f][layer-1]; - float max = maxVals[f][layer-1]; - features[f] = (features[f] - min) / (max-min); - } - features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); - - List sectionList = getSectionList(layer, sector); - for(int section : sectionList){ - for(int f = 0; f < NFEATURES; f++){ - x[section - 1][nHits[section - 1]][f] = features[f]; - } - - mask[section - 1][nHits[section - 1]] = 1.0f; - - maps[section - 1].put(nHits[section - 1], hit); - - nHits[section - 1]++; - if(nHits[section - 1] == MAX_HITS) { - Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); - return true; - } - } - } - - int maxIndex_BST = Integer.MIN_VALUE; - for (int i = 0; i < nHitsLayerSectorStrip_BST.length; i++) { - for (int j = 0; j < nHitsLayerSectorStrip_BST[i].length; j++) { - for (int k = 0; k < nHitsLayerSectorStrip_BST[i][j].length; k++) { - if (nHitsLayerSectorStrip_BST[i][j][k] > maxIndex_BST) { - maxIndex_BST = nHitsLayerSectorStrip_BST[i][j][k]; - } - } - } - } - int maxIndex_BMT = Integer.MIN_VALUE; - for (int i = 0; i < nHitsLayerSectorStrip_BMT.length; i++) { - for (int j = 0; j < nHitsLayerSectorStrip_BMT[i].length; j++) { - for (int k = 0; k < nHitsLayerSectorStrip_BMT[i][j].length; k++) { - if (nHitsLayerSectorStrip_BMT[i][j][k] > maxIndex_BMT) { - maxIndex_BMT = nHitsLayerSectorStrip_BMT[i][j][k]; - } - } - } - } - int maxIndex = Math.max(maxIndex_BST, maxIndex_BMT); - - // AI-based hit classification. - // Hit status is encoded in a single byte: - // bits [0–2] : noise flags for sections 1–3 (1 = noise, 0 = signal) - // bits [3–5] : hit presence for sections 1–3 (1 = hit present, 0 = no hit) - byte[][][][] statuses = new byte[NLAYERS][18][1152][maxIndex+1]; - for(int s = 0; s < NSECTIONS; s++){ - CVTInput input = new CVTInput(x[s],mask[s]); - try{ - Predictor predictor = predictors[s].take(); - - try{ - float[][] preds = predictor.predict(input); - for(int i = 0; i < nHits[s]; i++){ - Hit hit = maps[s].get(i); - statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << (s+3)); - if(preds[i][0] < thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << s); - } - } finally { - predictors[s].put(predictor); - } - - } catch (TranslateException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - // Update BST and BMT banks to record predictions - updateBanks(bst_bank, bmt_bank, statuses); - event.removeBank(BST_BANK); - event.appendBank(bst_bank); - event.removeBank(BMT_BANK); - event.appendBank(bmt_bank); - - return true; - } - - // Scaling per layer - private void initScaling(){ - - for(int l=0;l<12;l++) minVals[0][l]=1f; - - int[] stripMax={256,256,256,256,256,256,896,640,640,1024,768,1152}; - for(int l=0;l<12;l++) maxVals[0][l]=stripMax[l]; - - float[] xyMin={-7,-8,-10,-10,-15,-15,-15,-18,-18,-23,-23,-23}; - float[] xyMax={7,8,10,10,15,15,15,18,18,23,23,23}; - - for(int f=1;f<=4;f++){ - for(int l=0;l<12;l++){ - minVals[f][l]=xyMin[l]; - maxVals[f][l]=xyMax[l]; - } - } - - float[] zMin={-25,-25,-22,-22,-18,-18,-18,-21,-21,-21,-21,-21}; - float[] zMax={25,25,22,22,18,18,21,21,21,25,25,25}; - - for(int f=5;f<=6;f++){ - for(int l=0;l<12;l++){ - minVals[f][l]=zMin[l]; - maxVals[f][l]=zMax[l]; - } - } - - float[] sectorMax={11,11,15,15,19,19,3,3,3,3,3,3}; - - for(int l=0;l<12;l++){ - minVals[7][l]=1f; - maxVals[7][l]=sectorMax[l]; - } - - /* - float[] timeMin={0,0,0,0,0,0,4,4,4,4,4,4}; - float[] timeMax={511,511,511,511,511,511,436,436,436,436,436,436}; - - for(int l=0;l<12;l++){ - minVals[8][l]=timeMin[l]; - maxVals[8][l]=timeMax[l]; - } - */ - } - - // Set sections based on layer and sector of hits - // Some SVT hits are shared by sections1&2 or sections2&3 - private List getSectionList(int layer, int sector) { - List sectionList = new ArrayList<>(); - - if (layer >= 1 && layer <= 2) { - if (sector >= 1 && sector <= 4) sectionList.add(1); - if (sector >= 4 && sector <= 8) sectionList.add(2); - if (sector >= 8 && sector <= 10) sectionList.add(3); - } - else if (layer >= 3 && layer <= 4) { - if (sector >= 1 && sector <= 6) sectionList.add(1); - if (sector >= 6 && sector <= 10) sectionList.add(2); - if (sector >= 10 && sector <= 14) sectionList.add(3); - } - else if (layer >= 5 && layer <= 6) { - if (sector >= 1 && sector <= 7) sectionList.add(1); - if (sector >= 7 && sector <= 13) sectionList.add(2); - if (sector >= 14 && sector <= 18) sectionList.add(3); - } - else { - if (sector == 1) sectionList.add(1); - if (sector == 2) sectionList.add(2); - if (sector == 3) sectionList.add(3); - } - - return sectionList; - } - - // -------- Update BST & BMT banks -------- - private void updateBanks(DataBank bst_bank, DataBank bmt_bank, byte[][][][] statuses) { - int[][][] nHitsLayerSectorStrip_BST = new int[6][18][256]; // Number of hits in a strip - for (int row=0; row0; - } - - public boolean kfBeamSpot() { - return this.beamSpotConstraint==2; - } - - /** - * @return the docacut - */ - public double getDocacut() { - return docacut; - } - - /** - * @param docacut the docacut to set - */ - public void setDocacut(double docacut) { - this.docacut = docacut; - } - - /** - * @return the docacutsum - */ - public double getDocacutsum() { - return docacutsum; - } - - /** - * @return the svtmaxclussize - */ - public int getSvtmaxclussize() { - return svtmaxclussize; - } - - /** - * @param svtmaxclussize the svtmaxclussize to set - */ - public void setSvtmaxclussize(int svtmaxclussize) { - this.svtmaxclussize = svtmaxclussize; - } - - /** - * @return the bmtcmaxclussize - */ - public int getBmtcmaxclussize() { - return bmtcmaxclussize; - } - - /** - * @param bmtcmaxclussize the bmtcmaxclussize to set - */ - public void setBmtcmaxclussize(int bmtcmaxclussize) { - this.bmtcmaxclussize = bmtcmaxclussize; - } - - /** - * @return the bmtzmaxclussize - */ - public int getBmtzmaxclussize() { - return bmtzmaxclussize; - } - - /** - * @param bmtzmaxclussize the bmtzmaxclussize to set - */ - public void setBmtzmaxclussize(int bmtzmaxclussize) { - this.bmtzmaxclussize = bmtzmaxclussize; - } - - @Override - public boolean processDataEvent(DataEvent event) { - - Swim swimmer = new Swim(); - - int run = this.getRun(event); - - IndexedTable svtStatus = this.getConstantsManager().getConstants(run, "/calibration/svt/status"); - IndexedTable svtLorentz = this.getConstantsManager().getConstants(run, "/calibration/svt/lorentz_angle"); - IndexedTable bmtStatus = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_status"); - IndexedTable bmtTime = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_time"); - IndexedTable bmtVoltage = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_voltage"); - IndexedTable bmtStripVoltage = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_strip_voltage"); - IndexedTable bmtStripThreshold = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_strip_voltage_thresholds"); - IndexedTable beamPos = this.getConstantsManager().getConstants(run, "/geometry/beam/position"); - IndexedTable adcStatus = this.getConstantsManager().getConstants(run, "/calibration/svt/adcstatus"); - - Geometry.getInstance().initialize(this.getConstantsManager().getVariation(), run, svtLorentz, bmtVoltage); - - CVTReconstruction reco = new CVTReconstruction(swimmer); - - List> hits = reco.readHits(event, svtStatus, bmtStatus, bmtTime, - bmtStripVoltage, bmtStripThreshold, - adcStatus); - - - List banks = new ArrayList<>(); - - banks.add(RecoBankWriter.fillSVTHitBank(event, hits.get(0), this.getSvtHitBank())); - banks.add(RecoBankWriter.fillBMTHitBank(event, hits.get(1), this.getBmtHitBank())); - - event.appendBanks(banks.toArray(new DataBank[0])); - - - return true; - } - - - public void loadConfiguration() { - - // general (pass-independent) settings - if (this.getEngineConfigString("cosmics")!=null) - this.isCosmics = Boolean.valueOf(this.getEngineConfigString("cosmics")); - - if (this.getEngineConfigString("svtOnly")!=null) - this.svtOnly = Boolean.valueOf(this.getEngineConfigString("svtOnly")); - - if (this.getEngineConfigString("excludeLayers")!=null) - this.excludeLayers = this.getEngineConfigString("excludeLayers"); - - if (this.getEngineConfigString("excludeBMTLayers")!=null) - this.excludeBMTLayers = this.getEngineConfigString("excludeBMTLayers"); - - if (this.getEngineConfigString("removeRegion")!=null) - this.removeRegion = Integer.valueOf(this.getEngineConfigString("removeRegion")); - - if (this.getEngineConfigString("beamSpotConst")!=null) - this.beamSpotConstraint = Integer.valueOf(this.getEngineConfigString("beamSpotConst")); - - if (this.getEngineConfigString("beamSpotRadius")!=null) - this.beamSpotRadius = Double.valueOf(this.getEngineConfigString("beamSpotRadius")); - - if(this.getEngineConfigString("targetMat")!=null) - this.targetMaterial = this.getEngineConfigString("targetMat"); - - if(this.getEngineConfigString("elossPreCorrection")!=null) - this.elossPrecorrection = Boolean.parseBoolean(this.getEngineConfigString("elossPreCorrection")); - - if(this.getEngineConfigString("svtSeeding")!=null) - this.svtSeeding = Boolean.parseBoolean(this.getEngineConfigString("svtSeeding")); - - if(this.getEngineConfigString("timeCuts")!=null) - this.timeCuts = Boolean.parseBoolean(this.getEngineConfigString("timeCuts")); - - if(this.getEngineConfigString("hvCuts")!=null) - this.hvCuts = Boolean.parseBoolean(this.getEngineConfigString("hvCuts")); - - if(this.getEngineConfigString("useSVTTimingCuts")!=null) - this.useSVTTimingCuts = Boolean.parseBoolean(this.getEngineConfigString("useSVTTimingCuts")); - - if(this.getEngineConfigString("removeOverlappingSeeds")!=null) - this.removeOverlappingSeeds = Boolean.parseBoolean(this.getEngineConfigString("removeOverlappingSeeds")); - - if(this.getEngineConfigString("flagSeeds")!=null) - this.flagSeeds = Boolean.parseBoolean(this.getEngineConfigString("flagSeeds")); - - if(this.getEngineConfigString("gemcIgnBMT0ADC")!=null) - this.gemcIgnBMT0ADC = Boolean.parseBoolean(this.getEngineConfigString("gemcIgnBMT0ADC")); - - if(this.getEngineConfigString("KFfailRecovery")!=null) - this.KFfailRecovery = Boolean.parseBoolean(this.getEngineConfigString("KFfailRecovery")); - - if(this.getEngineConfigString("KFfailRecovMisCls")!=null) - this.KFfailRecovMisCls = Boolean.parseBoolean(this.getEngineConfigString("KFfailRecovMisCls")); - - if (this.getEngineConfigString("matLib")!=null) - this.matrixLibrary = this.getEngineConfigString("matLib"); - - // service dependent configuration settings - if(this.getEngineConfigString("elossPid")!=null) - this.pid = Integer.parseInt(this.getEngineConfigString("elossPid")); - - if (this.getEngineConfigString("kfFilterOn")!=null) - this.kfFilterOn = Boolean.valueOf(this.getEngineConfigString("kfFilterOn")); - - if (this.getEngineConfigString("initFromMC")!=null) - this.initFromMc = Boolean.valueOf(this.getEngineConfigString("initFromMC")); - - if (this.getEngineConfigString("useOnlyTruthHits")!=null) - this.useOnlyTruth = Boolean.valueOf(this.getEngineConfigString("useOnlyTruthHits")); - - if (this.getEngineConfigString("useSVTLinkerSeeder")!=null) - this.useSVTLinkerSeeder = Boolean.valueOf(this.getEngineConfigString("useSVTLinkerSeeder")); - - if (this.getEngineConfigString("kfIterations")!=null) - this.kfIterations = Integer.valueOf(this.getEngineConfigString("kfIterations")); - - if (this.getEngineConfigString("docacut")!=null) - this.setDocacut((double) Double.valueOf(this.getEngineConfigString("docacut"))); - - if (this.getEngineConfigString("docacutsum")!=null) - this.setDocacutsum((double) Double.valueOf(this.getEngineConfigString("docacutsum"))); - - if (this.getEngineConfigString("svtmaxclussize")!=null) - this.setSvtmaxclussize((int) Integer.valueOf(this.getEngineConfigString("svtmaxclussize"))); - - if (this.getEngineConfigString("bmtcmaxclussize")!=null) - this.setBmtcmaxclussize((int) Integer.valueOf(this.getEngineConfigString("bmtcmaxclussize"))); - - if (this.getEngineConfigString("bmtzmaxclussize")!=null) - this.setBmtzmaxclussize((int) Integer.valueOf(this.getEngineConfigString("bmtzmaxclussize"))); - - if (this.getEngineConfigString("rcut")!=null) - this.rcut = Double.valueOf(this.getEngineConfigString("rcut")); - - if (this.getEngineConfigString("z0cut")!=null) - this.z0cut = Double.valueOf(this.getEngineConfigString("z0cut")); - - } - - - public void initConstantsTables() { - String[] tables = new String[]{ - "/calibration/svt/status", - "/calibration/svt/lorentz_angle", - "/calibration/mvt/bmt_time", - "/calibration/mvt/bmt_status", - "/calibration/mvt/bmt_voltage", - "/calibration/mvt/bmt_strip_voltage", - "/calibration/mvt/bmt_strip_voltage_thresholds", - "/geometry/beam/position", - "/calibration/svt/adcstatus" - }; - requireConstants(Arrays.asList(tables)); - this.getConstantsManager().setVariation("default"); - } - - public void setSvtHitBank(String bstHitBank) { - this.svtHitBank = bstHitBank; - } - - public void setBmtHitBank(String bmtHitBank) { - this.bmtHitBank = bmtHitBank; - } - - public String getSvtHitBank() { - return svtHitBank; - } - - public String getBmtHitBank() { - return bmtHitBank; - } - - - public void printConfiguration() { - - System.out.println("["+this.getName()+"] run with cosmics setting set to "+Constants.getInstance().isCosmics); - System.out.println("["+this.getName()+"] run with SVT only set to "+Constants.getInstance().svtOnly); - if(this.excludeLayers!=null) - System.out.println("["+this.getName()+"] run with layers "+this.excludeLayers+" excluded in fit, based on yaml"); - if(this.excludeBMTLayers!=null) - System.out.println("["+this.getName()+"] run with BMT layers "+this.getEngineConfigString("excludeBMTLayers")+" excluded"); - if(this.removeRegion>0) - System.out.println("["+this.getName()+"] run with region "+this.getEngineConfigString("removeRegion")+" removed"); - System.out.println("["+this.getName()+"] run with beamSpotConst set to "+Constants.getInstance().beamSpotConstraint+ " (0=no-constraint, 1=seed only, 2=seed and KF)"); - System.out.println("["+this.getName()+"] run with beam spot size set to "+Constants.getInstance().getBeamRadius()); - System.out.println("["+this.getName()+"] Target material set to "+ Constants.getInstance().getTargetType()); - System.out.println("["+this.getName()+"] Pre-Eloss correction set to " + Constants.getInstance().preElossCorrection); - System.out.println("["+this.getName()+"] run SVT-based seeding set to "+ Constants.getInstance().svtSeeding); - System.out.println("["+this.getName()+"] run BMT timing cuts set to "+ Constants.getInstance().timeCuts); - System.out.println("["+this.getName()+"] run BMT HV masks "+ Constants.getInstance().bmtHVCuts); - System.out.println("["+this.getName()+"] run with matLib "+ Constants.getInstance().KFMatrixLibrary.toString() + " library"); - System.out.println("["+this.getName()+"] ELoss mass set for particle "+ pid); - System.out.println("["+this.getName()+"] run with Kalman-Filter status set to "+this.kfFilterOn); - System.out.println("["+this.getName()+"] initialize KF from true MC information "+this.initFromMc); - System.out.println("["+this.getName()+"] number of KF iterations set to "+this.kfIterations); - System.out.println("["+this.getName()+"] SLA doca cut "+this.docacut); - System.out.println("["+this.getName()+"] SLA docasum cut "+this.docacutsum); - System.out.println("["+this.getName()+"] max svt cluster size "+this.getSvtmaxclussize()); - System.out.println("["+this.getName()+"] max bmt-c cluster size "+this.getBmtcmaxclussize()); - System.out.println("["+this.getName()+"] max btm-z cluster size "+this.getBmtzmaxclussize()); - System.out.println("["+this.getName()+"] helix radius cut (mm) "+this.rcut); - System.out.println("["+this.getName()+"] z0 cut (mm from target edges) "+this.z0cut); - - - } - - - -} From 1eda7e922c8574ad8cc52cce76a38db3a4aa481c Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 27 Jul 2026 17:44:29 -0400 Subject: [PATCH 09/14] feat: override JVM args --- bin/hipoToNpz | 6 ++++-- .../clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java | 6 +----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/bin/hipoToNpz b/bin/hipoToNpz index da090d90af..055ee4e738 100755 --- a/bin/hipoToNpz +++ b/bin/hipoToNpz @@ -2,9 +2,11 @@ . `dirname $0`/../libexec/env.sh +split_cli $@ + export MALLOC_ARENA_MAX=1 -java ${JAVA_OPTS-} -Xmx1536m -Xms1024m -XX:+UseSerialGC \ +java ${JAVA_OPTS-} -Xmx1536m -Xms1024m -XX:+UseSerialGC ${jvm_options[@]} \ -cp ${COATJAVA_CLASSPATH:-''} \ org.jlab.io.hipo.HipoToNpz \ - $* + ${class_options[@]} diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java index a77b1364df..8aa74148a7 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java @@ -1,7 +1,3 @@ -/* - * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license - * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template - */ package org.jlab.io.hipo; import java.io.BufferedOutputStream; @@ -963,4 +959,4 @@ private static final class DoubleList { double[] toArray() { return Arrays.copyOf(data, size); } private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } } -} \ No newline at end of file +} From 1d9927f92c734f1195d82eb57d0cf8df6107a784 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 27 Jul 2026 18:05:26 -0400 Subject: [PATCH 10/14] feat: default schame dir --- .../main/java/org/jlab/io/hipo/HipoToNpz.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java index 8aa74148a7..b83c7a0ed7 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java @@ -25,6 +25,8 @@ import org.jlab.io.base.DataBank; import org.jlab.io.base.DataEvent; +import org.jlab.utils.system.ClasUtilsFile; + /** * * @author veronique @@ -98,7 +100,7 @@ private CliOptions(File input, File output, File schemaDir, Set selected } static CliOptions parse(String[] args) throws Exception { - if (args.length < 4) { + if (args.length < 2) { printUsageAndExit(); } @@ -149,7 +151,7 @@ static CliOptions parse(String[] args) throws Exception { } if (schemaDir == null) { - throw new IllegalArgumentException("--schema-dir is required"); + schemaDir = new File(ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/bankdefs/hipo4")); } if (!schemaDir.isDirectory()) { throw new IllegalArgumentException("Schema directory not found: " + schemaDir.getAbsolutePath()); @@ -178,8 +180,20 @@ private static Set readBankNames(File bankFile) throws IOException { } private static void printUsageAndExit() { - System.err.println("Usage:"); - System.err.println(" java org.jlab.io.hipo.HipoToNpz --schema-dir [bank1,bank2,...|*] [--bank-file banks.txt]"); + System.err.println("USAGE:"); + System.err.println(" hipoToNpz [OPTIONS...] [bank1,bank2,...|*]"); + System.err.println(""); + System.err.println("Specify a comma-delimited list of banks, otherwise it will use all banks found in the HIPO file"); + System.err.println(""); + System.err.println("OPTIONS:"); + System.err.println(" --bank-file FILE a file with one bank name per line, '#' comments allowed;"); + System.err.println(" both comma list and `--bank-file` may be used together"); + System.err.println(" --schema-dir DIR use a custom schema directory"); + System.err.println(""); + System.err.println("EXAMPLES:"); + System.err.println("* hipoToNpz input.hipo output.npz"); + System.err.println("* hipoToNpz input.hipo output.npz BST::adc,RUN::config"); + System.err.println("* hipoToNpz input.hipo output.npz --bank-file banks.txt"); System.exit(2); } } From 2e5347eafdc4d09ebf12b36725a5c2ba94e31e9a Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 27 Jul 2026 18:24:02 -0400 Subject: [PATCH 11/14] feat: ci --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++ HipoToNpz_README | 10 --------- list_npz_banks.py => bin/hipoToNpz-dump | 6 ++--- 3 files changed, 33 insertions(+), 13 deletions(-) delete mode 100644 HipoToNpz_README rename list_npz_banks.py => bin/hipoToNpz-dump (97%) mode change 100644 => 100755 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57a2afe9eb..58ec95ed4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,7 +229,13 @@ jobs: run: | ls -lhtr ./bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 500 -c ./clara -o ./tmp ./clas_018779.evio.00001 + mv -v tmp/rec_clas_018779.evio.00001 rec.hipo ls -lhtr + - uses: actions/upload-artifact@v7 + with: + name: test_clara_result + retention-days: 1 + path: rec.hipo test_coatjava: needs: [ build ] @@ -329,6 +335,30 @@ jobs: - name: test run-groovy run: coatjava/bin/run-groovy validation/advanced-tests/test-run-groovy.groovy + test_qcddat: + needs: test_clara + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Set up JDK + uses: actions/setup-java@v5.5.0 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: ${{ env.java_distribution }} + cache: maven + - uses: actions/download-artifact@v8 + with: + name: test_clara_result + - uses: actions/download-artifact@v8 + with: + name: build_ubuntu-latest + - name: untar build + run: tar xzvf coatjava.tar.gz + - name: hipoToNpz + run: ./coatjava/bin/hipoToNpz rec.hipo rec.npz RUN::config,REC::Event,REC::Particle + - name: hipoToNpz-dump + run: ./coatjava/bin/hipoToNpz-dump rec.npz 1 + # documentation ############################################################################# diff --git a/HipoToNpz_README b/HipoToNpz_README deleted file mode 100644 index 8f6ac0517f..0000000000 --- a/HipoToNpz_README +++ /dev/null @@ -1,10 +0,0 @@ -To convert a hipo file to npz format for a given set of banks: after compiling coatjava, cd coatjava, run the script: - * for all available banks: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 - * for BST::adc,RUN::config banks: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 BST::adc,RUN::config - * for banks listed in banks.txt: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 --bank-file banks.txt - -To inspect the npz file for n events: - * python3 list_npz_banks.py /pathTo/MyFile.npz n -Or - * python list_npz_banks.py /pathTo/MyFile.npz n - diff --git a/list_npz_banks.py b/bin/hipoToNpz-dump old mode 100644 new mode 100755 similarity index 97% rename from list_npz_banks.py rename to bin/hipoToNpz-dump index 930e930a9d..7ab7a09174 --- a/list_npz_banks.py +++ b/bin/hipoToNpz-dump @@ -1,10 +1,10 @@ #!/usr/bin/env python3 """ -Show values for a few events from all banks in an NPZ produced by HipoToNpz. +Show values for a few events from all banks in an NPZ produced by hipoToNpz. Usage: - python list_npz_banks.py file.npz - python list_npz_banks.py file.npz 3 + python hipoToNpz-dump file.npz + python hipoToNpz-dump file.npz 3 """ from __future__ import annotations From cc534293eeca357102f71a9f34e452d9ade9f5fa Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 27 Jul 2026 18:39:35 -0400 Subject: [PATCH 12/14] ci: ls --- .github/workflows/ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58ec95ed4e..e7fe6034ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,12 +225,13 @@ jobs: run: | tar xzvf clara.tar.gz tar xzvf coatjava.tar.gz + - run: ls - name: run test - run: | - ls -lhtr - ./bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 500 -c ./clara -o ./tmp ./clas_018779.evio.00001 - mv -v tmp/rec_clas_018779.evio.00001 rec.hipo - ls -lhtr + run: ./bin/run-clara -y ./etc/services/rgd-clarode.yml -t 4 -n 500 -c ./clara -o ./tmp ./clas_018779.evio.00001 + - name: ls tmp + run: ls -lhtr tmp + - name: rename + run: mv -v tmp/rec_clas_018779.evio.00001 rec.hipo - uses: actions/upload-artifact@v7 with: name: test_clara_result From 9f04a552e41b2977471a9068f026a039d96bb96a Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 27 Jul 2026 19:14:00 -0400 Subject: [PATCH 13/14] ci: .hipo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7fe6034ab..fd98d39c12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,7 +231,7 @@ jobs: - name: ls tmp run: ls -lhtr tmp - name: rename - run: mv -v tmp/rec_clas_018779.evio.00001 rec.hipo + run: mv -v tmp/rec_clas_018779.evio.00001.hipo rec.hipo - uses: actions/upload-artifact@v7 with: name: test_clara_result From 96b371dc284b6adfb78ad06aea5cdecc1f6c2e7a Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 28 Jul 2026 08:42:33 -0400 Subject: [PATCH 14/14] refactor: hipoToNpz -> hipo2npz --- .github/workflows/ci.yml | 8 ++++---- bin/{hipoToNpz => hipo2npz} | 2 +- bin/{hipoToNpz-dump => hipo2npz-dump} | 6 +++--- .../io/hipo/{HipoToNpz.java => Hipo2Npz.java} | 16 ++++++++-------- 4 files changed, 16 insertions(+), 16 deletions(-) rename bin/{hipoToNpz => hipo2npz} (87%) rename bin/{hipoToNpz-dump => hipo2npz-dump} (98%) rename common-tools/clas-io/src/main/java/org/jlab/io/hipo/{HipoToNpz.java => Hipo2Npz.java} (98%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd98d39c12..0c12c99dec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -355,10 +355,10 @@ jobs: name: build_ubuntu-latest - name: untar build run: tar xzvf coatjava.tar.gz - - name: hipoToNpz - run: ./coatjava/bin/hipoToNpz rec.hipo rec.npz RUN::config,REC::Event,REC::Particle - - name: hipoToNpz-dump - run: ./coatjava/bin/hipoToNpz-dump rec.npz 1 + - name: hipo2npz + run: ./coatjava/bin/hipo2npz rec.hipo rec.npz RUN::config,REC::Event,REC::Particle + - name: hipo2npz-dump + run: ./coatjava/bin/hipo2npz-dump rec.npz 1 # documentation ############################################################################# diff --git a/bin/hipoToNpz b/bin/hipo2npz similarity index 87% rename from bin/hipoToNpz rename to bin/hipo2npz index 055ee4e738..b08769accd 100755 --- a/bin/hipoToNpz +++ b/bin/hipo2npz @@ -8,5 +8,5 @@ export MALLOC_ARENA_MAX=1 java ${JAVA_OPTS-} -Xmx1536m -Xms1024m -XX:+UseSerialGC ${jvm_options[@]} \ -cp ${COATJAVA_CLASSPATH:-''} \ - org.jlab.io.hipo.HipoToNpz \ + org.jlab.io.hipo.Hipo2Npz \ ${class_options[@]} diff --git a/bin/hipoToNpz-dump b/bin/hipo2npz-dump similarity index 98% rename from bin/hipoToNpz-dump rename to bin/hipo2npz-dump index 7ab7a09174..2e93ce16ad 100755 --- a/bin/hipoToNpz-dump +++ b/bin/hipo2npz-dump @@ -1,10 +1,10 @@ #!/usr/bin/env python3 """ -Show values for a few events from all banks in an NPZ produced by hipoToNpz. +Show values for a few events from all banks in an NPZ produced by hipo2npz. Usage: - python hipoToNpz-dump file.npz - python hipoToNpz-dump file.npz 3 + python hipo2npz-dump file.npz + python hipo2npz-dump file.npz 3 """ from __future__ import annotations diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java similarity index 98% rename from common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java rename to common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java index b83c7a0ed7..867d600293 100644 --- a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/Hipo2Npz.java @@ -47,13 +47,13 @@ * - if no banks are selected, all JSON files in the folder are parsed * * Example: after compiling coatjava, cd coatjava, run the script: - * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json + * ./bin/hipo2npz input.hipo output.npz --schema-dir /path/to/json * - * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json BST::adc,RUN::config + * ./bin/hipo2npz input.hipo output.npz --schema-dir /path/to/json BST::adc,RUN::config * - * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json --bank-file banks.txt + * ./bin/hipo2npz input.hipo output.npz --schema-dir /path/to/json --bank-file banks.txt */ -public class HipoToNpz { +public class Hipo2Npz { public static void main(String[] args) throws Exception { CliOptions options = CliOptions.parse(args); @@ -181,7 +181,7 @@ private static Set readBankNames(File bankFile) throws IOException { private static void printUsageAndExit() { System.err.println("USAGE:"); - System.err.println(" hipoToNpz [OPTIONS...] [bank1,bank2,...|*]"); + System.err.println(" hipo2npz [OPTIONS...] [bank1,bank2,...|*]"); System.err.println(""); System.err.println("Specify a comma-delimited list of banks, otherwise it will use all banks found in the HIPO file"); System.err.println(""); @@ -191,9 +191,9 @@ private static void printUsageAndExit() { System.err.println(" --schema-dir DIR use a custom schema directory"); System.err.println(""); System.err.println("EXAMPLES:"); - System.err.println("* hipoToNpz input.hipo output.npz"); - System.err.println("* hipoToNpz input.hipo output.npz BST::adc,RUN::config"); - System.err.println("* hipoToNpz input.hipo output.npz --bank-file banks.txt"); + System.err.println("* hipo2npz input.hipo output.npz"); + System.err.println("* hipo2npz input.hipo output.npz BST::adc,RUN::config"); + System.err.println("* hipo2npz input.hipo output.npz --bank-file banks.txt"); System.exit(2); } }