From 676bc326ad142c90159a8ef5582ec56631be7434 Mon Sep 17 00:00:00 2001 From: tongtongcao Date: Thu, 12 Mar 2026 12:07:56 -0400 Subject: [PATCH 1/9] develop engine for CVT denoising --- reconstruction/cvt/pom.xml | 9 + .../rec/cvt/services/CVTDenoiseEngine.java | 533 ++++++++++++++++++ 2 files changed, 542 insertions(+) create mode 100644 reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java diff --git a/reconstruction/cvt/pom.xml b/reconstruction/cvt/pom.xml index 50e7eecd3a..7a9db52551 100644 --- a/reconstruction/cvt/pom.xml +++ b/reconstruction/cvt/pom.xml @@ -79,6 +79,11 @@ clas-geometry 14.1.2-SNAPSHOT + + org.jlab.clas12.detector + clas12detector-ai + 13.7.1-SNAPSHOT + org.jlab.clas12.detector clas12detector-eb @@ -95,6 +100,10 @@ junit junit + + ai.djl + api + diff --git a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java new file mode 100644 index 0000000000..70af041ea8 --- /dev/null +++ b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java @@ -0,0 +1,533 @@ +package org.jlab.rec.cvt.services; + +import ai.djl.MalformedModelException; +import java.nio.file.Paths; +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDList; +import ai.djl.ndarray.NDManager; +import ai.djl.ndarray.types.Shape; +import ai.djl.repository.zoo.Criteria; +import ai.djl.repository.zoo.ZooModel; +import ai.djl.training.util.ProgressBar; +import ai.djl.translate.Translator; +import ai.djl.translate.TranslatorContext; +import ai.djl.translate.Batchifier; +import ai.djl.inference.Predictor; +import ai.djl.repository.zoo.ModelNotFoundException; +import ai.djl.translate.TranslateException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.jlab.clas.reco.ReconstructionEngine; +import org.jlab.io.base.DataBank; +import org.jlab.io.base.DataEvent; +import org.jlab.utils.system.ClasUtilsFile; +import org.jlab.geom.prim.Line3D; +import org.jlab.geom.prim.Arc3D; +import org.jlab.geom.prim.Point3D; +import org.jlab.rec.cvt.Geometry; +import org.jlab.clas.swimtools.Swim; +import org.jlab.rec.cvt.Constants; +import org.jlab.service.ai.PredictorPool; +import org.jlab.utils.groups.IndexedTable; + +/* ----------------------------------------------- + Input class + ----------------------------------------------- */ +class CVTInput { + + float[][] data; // [450][9] + float[] mask; // [450] + + public CVTInput(float[][] data, float[] mask) { + this.data = data; + this.mask = mask; + } +} + +/* ----------------------------------------------- + DJL Translator + ----------------------------------------------- */ +class CVTTranslator implements Translator { + + @Override + public NDList processInput(TranslatorContext ctx, CVTInput input) { + + NDManager manager = ctx.getNDManager(); + + NDArray x = manager.create(input.data).reshape(1, 450, 9); + NDArray mask = manager.create(input.mask).reshape(1, 450); + + return new NDList(x, mask); + } + + @Override + public float[][] processOutput(TranslatorContext ctx, NDList list) { + + NDArray output = list.singletonOrThrow(); // shape: (1, 450, C) or (1, 450) + output = output.squeeze(0); // remove batch dim -> (450, C) + + float[] flat = output.toFloatArray(); + long[] shape = output.getShape().getShape(); + + int dim0 = (int) shape[0]; + int dim1 = shape.length > 1 ? (int) shape[1] : 1; + + float[][] result = new float[dim0][dim1]; + + for(int i=0;i[] criterias; + private ZooModel[] model; + private PredictorPool[] predictors; + + public CVTDenoiseEngine() { + super("CVTDenoiseEngine","Tongtong","1.0"); + } + + + @Override + public boolean init() { + this.initConstantsTables(); + + criterias = new Criteria[NSECTIONS]; + model = new ZooModel[NSECTIONS]; + predictors = new PredictorPool[NSECTIONS]; + + initScaling(); // Initial minVals and maxVals + + System.setProperty("ai.djl.pytorch.num_interop_threads", "1"); + System.setProperty("ai.djl.pytorch.num_threads", "1"); + System.setProperty("ai.djl.pytorch.graph_optimizer", "false"); + + + // Load models and set predictor pools + for(int i = 0; i < NSECTIONS; i++){ + if (getEngineConfigString(CONF_THRESHOLDS[i]) != null) + thresholds[i] = Float.parseFloat(getEngineConfigString(CONF_THRESHOLDS[i])); + if (getEngineConfigString(CONF_MODEL_FILES[i]) != null) + modelFiles[i] = getEngineConfigString(CONF_MODEL_FILES[i]); + + try { + String modelPath = ClasUtilsFile.getResourceDir("CLAS12DIR", "etc/data/nnet/cvtdn/" + modelFiles[i]); + + CVTTranslator translator = new CVTTranslator(); + criterias[i] = Criteria.builder() + .setTypes(CVTInput.class, float[][].class) + .optModelPath(Paths.get(modelPath)) + .optEngine("PyTorch") + .optTranslator(translator) + .optProgress(new ProgressBar()) + .build(); + + model[i] = criterias[i].loadModel(); + + int threads = Integer.parseInt(getEngineConfigString(CONF_THREADS,"16")); + predictors[i] = new PredictorPool(threads, model[i]); + } catch (NullPointerException | MalformedModelException | IOException | ModelNotFoundException ex) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, null, ex); + return false; + } + } + + return true; + } + + @Override + public boolean processDataEvent(DataEvent event) { + + Swim swimmer = new Swim(); + + int run = this.getRun(event); + + IndexedTable svtLorentz = this.getConstantsManager().getConstants(run, "/calibration/svt/lorentz_angle"); + IndexedTable bmtVoltage = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_voltage"); + + Geometry.getInstance().initialize(this.getConstantsManager().getVariation(), run, svtLorentz, bmtVoltage); + + if (!event.hasBank(BST_BANK)) return true; + if (!event.hasBank(BMT_BANK)) return true; + + DataBank bst_bank = event.getBank(BST_BANK); + DataBank bmt_bank = event.getBank(BMT_BANK); + + float[][][] x = new float[NSECTIONS][MAX_HITS][NFEATURES]; + float[][] mask = new float[NSECTIONS][MAX_HITS]; + + int[] nHits = {0, 0, 0}; + + Map[] maps = new HashMap[NSECTIONS]; + for (int i = 0; i < NSECTIONS; i++) { + maps[i] = new HashMap<>(); + } + + // Read BST bank and set input for models + int[][][] nHitsLayerSectorStrip_BST = new int[6][18][256]; // Number of hits in a strip + for(int i = 0; i < bst_bank.rows(); i++){ + int sector = bst_bank.getByte("sector", i); + int layer = bst_bank.getByte("layer", i); + int strip = bst_bank.getShort("component",i); + int order = bst_bank.getByte("order", i); + int adc = bst_bank.getInt("ADC", i); + + Hit hit = new Hit(layer, sector, strip, nHitsLayerSectorStrip_BST[layer-1][sector-1][strip-1]++); + + if(order == 0 || order == 10){ + Line3D line = Geometry.getInstance().getSVT().getStrip(layer, sector, strip); + float[] features = {strip, (float)(line.origin().x()/10.), (float)(line.end().x()/10.), (float)(line.origin().y()/10.), + (float)(line.end().y()/10.), (float)(line.origin().z()/10.), (float)(line.end().z()/10.), sector, layer}; // unit conversion from mm to cm for end points + + for(int f = 0; f < NFEATURES - 1; f++){ + float min = minVals[f][layer-1]; + float max = maxVals[f][layer-1]; + features[f] = (features[f] - min) / (max-min); + } + features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); + + List sectionList = getSectionList(layer, sector); + for(int section : sectionList){ + for(int f = 0; f < NFEATURES; f++){ + x[section - 1][nHits[section - 1]][f] = features[f]; + } + + mask[section - 1][nHits[section - 1]] = 1.0f; + + maps[section - 1].put(nHits[section - 1], hit); + + nHits[section - 1]++; + if(nHits[section - 1] == MAX_HITS) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); + return true; + } + } + } + } + + // Read BMT bank and set input for models + int[][][] nHitsLayerSectorStrip_BMT = new int[6][3][1152]; // Number of hits in a strip + for(int i = 0; i < bmt_bank.rows(); i++){ + int sector = bmt_bank.getByte("sector", i); + int layer = bmt_bank.getByte("layer", i); + int strip = bmt_bank.getShort("component",i); + int order = bmt_bank.getByte("order", i); + int adc = bst_bank.getInt("ADC", i); + + Hit hit = new Hit(layer+6, sector, strip, nHitsLayerSectorStrip_BMT[layer-1][sector-1][strip-1]++); + + if(order == 0 || order == 10){ + Point3D originPoint, endPoint; + int region = (layer + 1) / 2; // Get region number for a BMT layer; layers 1, 4, 6 for BMT-C layers, and layers 2, 3, 5 for BMT-Z + if(layer == 2 || layer ==3 || layer == 5) { + Line3D line = Geometry.getInstance().getBMT().getLCZstrip(region, sector, strip, swimmer); + originPoint = line.origin(); + endPoint = line.origin(); + } + else { + Arc3D arcLine = Geometry.getInstance().getBMT().getCstrip(region, sector, strip); + originPoint = arcLine.origin(); + endPoint = arcLine.end(); + } + + layer += 6; + float[] features = {strip, (float)(originPoint.x()/10.), (float)(endPoint.x()/10.), (float)(originPoint.y()/10.), + (float)(endPoint.y()/10.), (float)(originPoint.z()/10.), (float)(endPoint.z()/10.), sector, layer}; // unit conversion from mm to cm for end points + + for(int f = 0; f < NFEATURES - 1; f++){ + float min = minVals[f][layer-1]; + float max = maxVals[f][layer-1]; + features[f] = (features[f] - min) / (max-min); + } + features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); + + List sectionList = getSectionList(layer, sector); + for(int section : sectionList){ + for(int f = 0; f < NFEATURES; f++){ + x[section - 1][nHits[section - 1]][f] = features[f]; + } + + mask[section - 1][nHits[section - 1]] = 1.0f; + + maps[section - 1].put(nHits[section - 1], hit); + + nHits[section - 1]++; + if(nHits[section - 1] == MAX_HITS) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); + return true; + } + } + } + } + + // Apply models for prediction of hits + // Status for a hit with order of 0: + // 0: rejected + // 1: accepted by section 1 + // 2: accepted by section 2 + // 3: accepted by section 3 + // 12: accepted by sections 1 & 2; Note: a hit is shared by secton 1 & 2, and is accpcted by both section + // 23: accepted by sections 2 & 3; Note: a hit is shared by secton 2 & 3, and is accpcted by both section + int maxIndex_BST = Integer.MIN_VALUE; + for (int i = 0; i < nHitsLayerSectorStrip_BST.length; i++) { + for (int j = 0; j < nHitsLayerSectorStrip_BST[i].length; j++) { + for (int k = 0; k < nHitsLayerSectorStrip_BST[i][j].length; k++) { + if (nHitsLayerSectorStrip_BST[i][j][k] > maxIndex_BST) { + maxIndex_BST = nHitsLayerSectorStrip_BST[i][j][k]; + } + } + } + } + int maxIndex_BMT = Integer.MIN_VALUE; + for (int i = 0; i < nHitsLayerSectorStrip_BMT.length; i++) { + for (int j = 0; j < nHitsLayerSectorStrip_BMT[i].length; j++) { + for (int k = 0; k < nHitsLayerSectorStrip_BMT[i][j].length; k++) { + if (nHitsLayerSectorStrip_BMT[i][j][k] > maxIndex_BMT) { + maxIndex_BMT = nHitsLayerSectorStrip_BMT[i][j][k]; + } + } + } + } + int maxIndex = Math.max(maxIndex_BST, maxIndex_BMT); + + byte[][][][] statuses = new byte[NLAYERS][18][1152][maxIndex+1]; + for(int s = 0; s < NSECTIONS; s++){ + CVTInput input = new CVTInput(x[s],mask[s]); + try{ + Predictor predictor = predictors[s].take(); + + try{ + float[][] preds = predictor.predict(input); + for(int i = 0; i < nHits[s]; i++){ + Hit hit = maps[s].get(i); + byte status = statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()]; + if(preds[i][0] > thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] = (byte) (status * 10 + s + 1); + //if(statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] < 0 || statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] > 123) + // System.out.println(preds[i][0] + " " + s + " " + i + " " + hit.getLayer() + " " + hit.getSector() + " " + hit.getStrip() + " " + hit.getIndex() + " " + statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()]); + } + } finally { + predictors[s].put(predictor); + } + + } catch (TranslateException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + // Update BST and BMT banks to record predictions + updateBanks(bst_bank, bmt_bank, statuses); + event.removeBank(BST_BANK); + event.appendBank(bst_bank); + event.removeBank(BMT_BANK); + event.appendBank(bmt_bank); + + return true; + } + + // Scaling per layer + private void initScaling(){ + + for(int l=0;l<12;l++) minVals[0][l]=1f; + + int[] stripMax={256,256,256,256,256,256,896,640,640,1024,768,1152}; + for(int l=0;l<12;l++) maxVals[0][l]=stripMax[l]; + + float[] xyMin={-7,-8,-10,-10,-15,-15,-15,-18,-18,-23,-23,-23}; + float[] xyMax={7,8,10,10,15,15,15,18,18,23,23,23}; + + for(int f=1;f<=4;f++){ + for(int l=0;l<12;l++){ + minVals[f][l]=xyMin[l]; + maxVals[f][l]=xyMax[l]; + } + } + + float[] zMin={-25,-25,-22,-22,-18,-18,-18,-21,-21,-21,-21,-21}; + float[] zMax={25,25,22,22,18,18,21,21,21,25,25,25}; + + for(int f=5;f<=6;f++){ + for(int l=0;l<12;l++){ + minVals[f][l]=zMin[l]; + maxVals[f][l]=zMax[l]; + } + } + + float[] sectorMax={11,11,15,15,19,19,3,3,3,3,3,3}; + + for(int l=0;l<12;l++){ + minVals[7][l]=1f; + maxVals[7][l]=sectorMax[l]; + } + + /* + float[] timeMin={0,0,0,0,0,0,4,4,4,4,4,4}; + float[] timeMax={511,511,511,511,511,511,436,436,436,436,436,436}; + + for(int l=0;l<12;l++){ + minVals[8][l]=timeMin[l]; + maxVals[8][l]=timeMax[l]; + } + */ + } + + // Set sections based on layer and sector of hits + // Some SVT hits are shared by sections1&2 or sections2&3 + private List getSectionList(int layer, int sector) { + List sectionList = new ArrayList<>(); + + if (layer >= 1 && layer <= 2) { + if (sector >= 1 && sector <= 4) sectionList.add(1); + if (sector >= 4 && sector <= 8) sectionList.add(2); + if (sector >= 8 && sector <= 10) sectionList.add(3); + } + else if (layer >= 3 && layer <= 4) { + if (sector >= 1 && sector <= 6) sectionList.add(1); + if (sector >= 6 && sector <= 10) sectionList.add(2); + if (sector >= 10 && sector <= 14) sectionList.add(3); + } + else if (layer >= 5 && layer <= 6) { + if (sector >= 1 && sector <= 7) sectionList.add(1); + if (sector >= 7 && sector <= 13) sectionList.add(2); + if (sector >= 14 && sector <= 18) sectionList.add(3); + } + else { + if (sector == 1) sectionList.add(1); + if (sector == 2) sectionList.add(2); + if (sector == 3) sectionList.add(3); + } + + return sectionList; + } + + // -------- Update BST & BMT banks -------- + private void updateBanks(DataBank bst_bank, DataBank bmt_bank, byte[][][][] statuses) { + int[][][] nHitsLayerSectorStrip_BST = new int[6][18][256]; // Number of hits in a strip + for (int row=0; row Date: Mon, 16 Mar 2026 11:56:15 -0400 Subject: [PATCH 2/9] CVT Hits reconstruction stand-alone service. --- .../jlab/rec/cvt/services/CVTHitEngine.java | 447 ++++++++++++++++++ 1 file changed, 447 insertions(+) create mode 100644 reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTHitEngine.java diff --git a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTHitEngine.java b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTHitEngine.java new file mode 100644 index 0000000000..625002a192 --- /dev/null +++ b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTHitEngine.java @@ -0,0 +1,447 @@ +package org.jlab.rec.cvt.services; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.jlab.clas.reco.ReconstructionEngine; +import org.jlab.clas.swimtools.Swim; +import org.jlab.io.base.DataBank; +import org.jlab.io.base.DataEvent; +import org.jlab.rec.cvt.Constants; +import org.jlab.rec.cvt.Geometry; +import org.jlab.rec.cvt.banks.RecoBankWriter; +import org.jlab.rec.cvt.hit.Hit; +import org.jlab.utils.groups.IndexedTable; + +/** + * Service to return reconstructed HITS + * + * @author ziegler + * + */ +public class CVTHitEngine extends ReconstructionEngine { + + + /** + * @param docacutsum the docacutsum to set + */ + public void setDocacutsum(double docacutsum) { + this.docacutsum = docacutsum; + } + + private int Run = -1; + + private String svtHitBank; + private String bmtHitBank; + private String bankPrefix = ""; + + // run-time options + private int pid = 0; + private int kfIterations = 5; + private boolean kfFilterOn = true; + private boolean initFromMc = false; + + // yaml setting passed to Constants class + private boolean isCosmics = false; + private boolean svtOnly = false; + private String excludeLayers = null; + private String excludeBMTLayers = null; + private int removeRegion = 0; + private int beamSpotConstraint = 2; + private double beamSpotRadius = 0.3; + private String targetMaterial = ""; + private boolean elossPrecorrection = true; + private boolean svtSeeding = true; + private boolean timeCuts = true; + private boolean hvCuts = false; + public boolean useSVTTimingCuts = false; + public boolean removeOverlappingSeeds = true; + public boolean flagSeeds = true; + public boolean gemcIgnBMT0ADC = false; + public boolean KFfailRecovery = true; + public boolean KFfailRecovMisCls = true; + private String matrixLibrary = "EJML"; + private boolean useOnlyTruth = false; + private boolean useSVTLinkerSeeder = true; + private double docacut = 0.75; + private double docacutsum = 1.15; + private int svtmaxclussize = 100; + private int bmtcmaxclussize = 100; + private int bmtzmaxclussize = 100; + private double rcut = 120.0; + private double z0cut = 10; + + public CVTHitEngine(String name) { + super(name, "ziegler", "6.0"); + } + + public CVTHitEngine() { + super("CVTHitEngine", "ziegler", "6.0"); + } + + + @Override + public boolean init() { + this.loadConfiguration(); + Constants.getInstance().initialize(this.getName(), + isCosmics, + svtOnly, + excludeLayers, + excludeBMTLayers, + removeRegion, + beamSpotConstraint, + beamSpotRadius, + targetMaterial, + elossPrecorrection, + svtSeeding, + timeCuts, + hvCuts, + useSVTTimingCuts, + removeOverlappingSeeds, + flagSeeds, + gemcIgnBMT0ADC, + KFfailRecovery, + KFfailRecovMisCls, + matrixLibrary, + useOnlyTruth, + useSVTLinkerSeeder, + docacut, + docacutsum, + svtmaxclussize, + bmtcmaxclussize, + bmtzmaxclussize, + rcut, + z0cut); + + this.initConstantsTables(); + this.registerBanks(); + this.printConfiguration(); + return true; + } + + public final void setOutputBankPrefix(String prefix) { + this.bankPrefix = prefix; + } + + public void registerBanks() { + String prefix = bankPrefix; + if(Constants.getInstance().isCosmics) prefix = "Rec"; + this.setBmtHitBank("BMT" + prefix + "::Hits"); + this.setSvtHitBank("BST" + prefix + "::Hits"); + super.registerOutputBank(this.bmtHitBank); + super.registerOutputBank(this.svtHitBank); + } + + public int getRun(DataEvent event) { + + if (event.hasBank("RUN::config") == false) { + System.err.println("RUN CONDITIONS NOT READ!"); + return 0; + } + + DataBank bank = event.getBank("RUN::config"); + int run = bank.getInt("run", 0); + if(Constants.getInstance().seedingDebugMode) { + System.out.println("EVENT "+bank.getInt("event", 0)); + } + return run; + } + + public int getPid() { + return pid; + } + + public int getKfIterations() { + return kfIterations; + } + + public boolean isKfFilterOn() { + return kfFilterOn; + } + + public boolean isInitFromMc() { + return initFromMc; + } + + public boolean seedBeamSpot() { + return this.beamSpotConstraint>0; + } + + public boolean kfBeamSpot() { + return this.beamSpotConstraint==2; + } + + /** + * @return the docacut + */ + public double getDocacut() { + return docacut; + } + + /** + * @param docacut the docacut to set + */ + public void setDocacut(double docacut) { + this.docacut = docacut; + } + + /** + * @return the docacutsum + */ + public double getDocacutsum() { + return docacutsum; + } + + /** + * @return the svtmaxclussize + */ + public int getSvtmaxclussize() { + return svtmaxclussize; + } + + /** + * @param svtmaxclussize the svtmaxclussize to set + */ + public void setSvtmaxclussize(int svtmaxclussize) { + this.svtmaxclussize = svtmaxclussize; + } + + /** + * @return the bmtcmaxclussize + */ + public int getBmtcmaxclussize() { + return bmtcmaxclussize; + } + + /** + * @param bmtcmaxclussize the bmtcmaxclussize to set + */ + public void setBmtcmaxclussize(int bmtcmaxclussize) { + this.bmtcmaxclussize = bmtcmaxclussize; + } + + /** + * @return the bmtzmaxclussize + */ + public int getBmtzmaxclussize() { + return bmtzmaxclussize; + } + + /** + * @param bmtzmaxclussize the bmtzmaxclussize to set + */ + public void setBmtzmaxclussize(int bmtzmaxclussize) { + this.bmtzmaxclussize = bmtzmaxclussize; + } + + @Override + public boolean processDataEvent(DataEvent event) { + + Swim swimmer = new Swim(); + + int run = this.getRun(event); + + IndexedTable svtStatus = this.getConstantsManager().getConstants(run, "/calibration/svt/status"); + IndexedTable svtLorentz = this.getConstantsManager().getConstants(run, "/calibration/svt/lorentz_angle"); + IndexedTable bmtStatus = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_status"); + IndexedTable bmtTime = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_time"); + IndexedTable bmtVoltage = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_voltage"); + IndexedTable bmtStripVoltage = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_strip_voltage"); + IndexedTable bmtStripThreshold = this.getConstantsManager().getConstants(run, "/calibration/mvt/bmt_strip_voltage_thresholds"); + IndexedTable beamPos = this.getConstantsManager().getConstants(run, "/geometry/beam/position"); + IndexedTable adcStatus = this.getConstantsManager().getConstants(run, "/calibration/svt/adcstatus"); + + Geometry.getInstance().initialize(this.getConstantsManager().getVariation(), run, svtLorentz, bmtVoltage); + + CVTReconstruction reco = new CVTReconstruction(swimmer); + + List> hits = reco.readHits(event, svtStatus, bmtStatus, bmtTime, + bmtStripVoltage, bmtStripThreshold, + adcStatus); + + + List banks = new ArrayList<>(); + + banks.add(RecoBankWriter.fillSVTHitBank(event, hits.get(0), this.getSvtHitBank())); + banks.add(RecoBankWriter.fillBMTHitBank(event, hits.get(1), this.getBmtHitBank())); + + event.appendBanks(banks.toArray(new DataBank[0])); + + + return true; + } + + + public void loadConfiguration() { + + // general (pass-independent) settings + if (this.getEngineConfigString("cosmics")!=null) + this.isCosmics = Boolean.valueOf(this.getEngineConfigString("cosmics")); + + if (this.getEngineConfigString("svtOnly")!=null) + this.svtOnly = Boolean.valueOf(this.getEngineConfigString("svtOnly")); + + if (this.getEngineConfigString("excludeLayers")!=null) + this.excludeLayers = this.getEngineConfigString("excludeLayers"); + + if (this.getEngineConfigString("excludeBMTLayers")!=null) + this.excludeBMTLayers = this.getEngineConfigString("excludeBMTLayers"); + + if (this.getEngineConfigString("removeRegion")!=null) + this.removeRegion = Integer.valueOf(this.getEngineConfigString("removeRegion")); + + if (this.getEngineConfigString("beamSpotConst")!=null) + this.beamSpotConstraint = Integer.valueOf(this.getEngineConfigString("beamSpotConst")); + + if (this.getEngineConfigString("beamSpotRadius")!=null) + this.beamSpotRadius = Double.valueOf(this.getEngineConfigString("beamSpotRadius")); + + if(this.getEngineConfigString("targetMat")!=null) + this.targetMaterial = this.getEngineConfigString("targetMat"); + + if(this.getEngineConfigString("elossPreCorrection")!=null) + this.elossPrecorrection = Boolean.parseBoolean(this.getEngineConfigString("elossPreCorrection")); + + if(this.getEngineConfigString("svtSeeding")!=null) + this.svtSeeding = Boolean.parseBoolean(this.getEngineConfigString("svtSeeding")); + + if(this.getEngineConfigString("timeCuts")!=null) + this.timeCuts = Boolean.parseBoolean(this.getEngineConfigString("timeCuts")); + + if(this.getEngineConfigString("hvCuts")!=null) + this.hvCuts = Boolean.parseBoolean(this.getEngineConfigString("hvCuts")); + + if(this.getEngineConfigString("useSVTTimingCuts")!=null) + this.useSVTTimingCuts = Boolean.parseBoolean(this.getEngineConfigString("useSVTTimingCuts")); + + if(this.getEngineConfigString("removeOverlappingSeeds")!=null) + this.removeOverlappingSeeds = Boolean.parseBoolean(this.getEngineConfigString("removeOverlappingSeeds")); + + if(this.getEngineConfigString("flagSeeds")!=null) + this.flagSeeds = Boolean.parseBoolean(this.getEngineConfigString("flagSeeds")); + + if(this.getEngineConfigString("gemcIgnBMT0ADC")!=null) + this.gemcIgnBMT0ADC = Boolean.parseBoolean(this.getEngineConfigString("gemcIgnBMT0ADC")); + + if(this.getEngineConfigString("KFfailRecovery")!=null) + this.KFfailRecovery = Boolean.parseBoolean(this.getEngineConfigString("KFfailRecovery")); + + if(this.getEngineConfigString("KFfailRecovMisCls")!=null) + this.KFfailRecovMisCls = Boolean.parseBoolean(this.getEngineConfigString("KFfailRecovMisCls")); + + if (this.getEngineConfigString("matLib")!=null) + this.matrixLibrary = this.getEngineConfigString("matLib"); + + // service dependent configuration settings + if(this.getEngineConfigString("elossPid")!=null) + this.pid = Integer.parseInt(this.getEngineConfigString("elossPid")); + + if (this.getEngineConfigString("kfFilterOn")!=null) + this.kfFilterOn = Boolean.valueOf(this.getEngineConfigString("kfFilterOn")); + + if (this.getEngineConfigString("initFromMC")!=null) + this.initFromMc = Boolean.valueOf(this.getEngineConfigString("initFromMC")); + + if (this.getEngineConfigString("useOnlyTruthHits")!=null) + this.useOnlyTruth = Boolean.valueOf(this.getEngineConfigString("useOnlyTruthHits")); + + if (this.getEngineConfigString("useSVTLinkerSeeder")!=null) + this.useSVTLinkerSeeder = Boolean.valueOf(this.getEngineConfigString("useSVTLinkerSeeder")); + + if (this.getEngineConfigString("kfIterations")!=null) + this.kfIterations = Integer.valueOf(this.getEngineConfigString("kfIterations")); + + if (this.getEngineConfigString("docacut")!=null) + this.setDocacut((double) Double.valueOf(this.getEngineConfigString("docacut"))); + + if (this.getEngineConfigString("docacutsum")!=null) + this.setDocacutsum((double) Double.valueOf(this.getEngineConfigString("docacutsum"))); + + if (this.getEngineConfigString("svtmaxclussize")!=null) + this.setSvtmaxclussize((int) Integer.valueOf(this.getEngineConfigString("svtmaxclussize"))); + + if (this.getEngineConfigString("bmtcmaxclussize")!=null) + this.setBmtcmaxclussize((int) Integer.valueOf(this.getEngineConfigString("bmtcmaxclussize"))); + + if (this.getEngineConfigString("bmtzmaxclussize")!=null) + this.setBmtzmaxclussize((int) Integer.valueOf(this.getEngineConfigString("bmtzmaxclussize"))); + + if (this.getEngineConfigString("rcut")!=null) + this.rcut = Double.valueOf(this.getEngineConfigString("rcut")); + + if (this.getEngineConfigString("z0cut")!=null) + this.z0cut = Double.valueOf(this.getEngineConfigString("z0cut")); + + } + + + public void initConstantsTables() { + String[] tables = new String[]{ + "/calibration/svt/status", + "/calibration/svt/lorentz_angle", + "/calibration/mvt/bmt_time", + "/calibration/mvt/bmt_status", + "/calibration/mvt/bmt_voltage", + "/calibration/mvt/bmt_strip_voltage", + "/calibration/mvt/bmt_strip_voltage_thresholds", + "/geometry/beam/position", + "/calibration/svt/adcstatus" + }; + requireConstants(Arrays.asList(tables)); + this.getConstantsManager().setVariation("default"); + } + + public void setSvtHitBank(String bstHitBank) { + this.svtHitBank = bstHitBank; + } + + public void setBmtHitBank(String bmtHitBank) { + this.bmtHitBank = bmtHitBank; + } + + public String getSvtHitBank() { + return svtHitBank; + } + + public String getBmtHitBank() { + return bmtHitBank; + } + + + public void printConfiguration() { + + System.out.println("["+this.getName()+"] run with cosmics setting set to "+Constants.getInstance().isCosmics); + System.out.println("["+this.getName()+"] run with SVT only set to "+Constants.getInstance().svtOnly); + if(this.excludeLayers!=null) + System.out.println("["+this.getName()+"] run with layers "+this.excludeLayers+" excluded in fit, based on yaml"); + if(this.excludeBMTLayers!=null) + System.out.println("["+this.getName()+"] run with BMT layers "+this.getEngineConfigString("excludeBMTLayers")+" excluded"); + if(this.removeRegion>0) + System.out.println("["+this.getName()+"] run with region "+this.getEngineConfigString("removeRegion")+" removed"); + System.out.println("["+this.getName()+"] run with beamSpotConst set to "+Constants.getInstance().beamSpotConstraint+ " (0=no-constraint, 1=seed only, 2=seed and KF)"); + System.out.println("["+this.getName()+"] run with beam spot size set to "+Constants.getInstance().getBeamRadius()); + System.out.println("["+this.getName()+"] Target material set to "+ Constants.getInstance().getTargetType()); + System.out.println("["+this.getName()+"] Pre-Eloss correction set to " + Constants.getInstance().preElossCorrection); + System.out.println("["+this.getName()+"] run SVT-based seeding set to "+ Constants.getInstance().svtSeeding); + System.out.println("["+this.getName()+"] run BMT timing cuts set to "+ Constants.getInstance().timeCuts); + System.out.println("["+this.getName()+"] run BMT HV masks "+ Constants.getInstance().bmtHVCuts); + System.out.println("["+this.getName()+"] run with matLib "+ Constants.getInstance().KFMatrixLibrary.toString() + " library"); + System.out.println("["+this.getName()+"] ELoss mass set for particle "+ pid); + System.out.println("["+this.getName()+"] run with Kalman-Filter status set to "+this.kfFilterOn); + System.out.println("["+this.getName()+"] initialize KF from true MC information "+this.initFromMc); + System.out.println("["+this.getName()+"] number of KF iterations set to "+this.kfIterations); + System.out.println("["+this.getName()+"] SLA doca cut "+this.docacut); + System.out.println("["+this.getName()+"] SLA docasum cut "+this.docacutsum); + System.out.println("["+this.getName()+"] max svt cluster size "+this.getSvtmaxclussize()); + System.out.println("["+this.getName()+"] max bmt-c cluster size "+this.getBmtcmaxclussize()); + System.out.println("["+this.getName()+"] max btm-z cluster size "+this.getBmtzmaxclussize()); + System.out.println("["+this.getName()+"] helix radius cut (mm) "+this.rcut); + System.out.println("["+this.getName()+"] z0 cut (mm from target edges) "+this.z0cut); + + + } + + + +} From cf2643ab00b6f36ea4502556f1e860a8208a8254 Mon Sep 17 00:00:00 2001 From: tongtongcao Date: Mon, 16 Mar 2026 17:56:52 -0400 Subject: [PATCH 3/9] update CVTDenoiseEngine with using BST::Hits and BMT::Hits as input --- etc/bankdefs/hipo4/bmt.json | 3 +- etc/bankdefs/hipo4/bst.json | 3 +- .../rec/cvt/services/CVTDenoiseEngine.java | 189 +++++++++--------- 3 files changed, 93 insertions(+), 102 deletions(-) diff --git a/etc/bankdefs/hipo4/bmt.json b/etc/bankdefs/hipo4/bmt.json index 616793bd1f..3d50008262 100644 --- a/etc/bankdefs/hipo4/bmt.json +++ b/etc/bankdefs/hipo4/bmt.json @@ -129,7 +129,8 @@ {"name":"trkingStat", "type":"B", "info":"tracking status"}, {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, - {"name":"status", "type":"B", "info":"status (0=good)"} + {"name":"status", "type":"B", "info":"status (0=good)"}, + {"name":"ai", "type":"B", "info":"3 bits corresponds to 3 sections; 1 = singal and 0 = noise"} ] }, { diff --git a/etc/bankdefs/hipo4/bst.json b/etc/bankdefs/hipo4/bst.json index 8d9c6c46ba..a45c3475b1 100644 --- a/etc/bankdefs/hipo4/bst.json +++ b/etc/bankdefs/hipo4/bst.json @@ -120,7 +120,8 @@ {"name":"trkingStat", "type":"B", "info":"tracking status"}, {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, - {"name":"status", "type":"B", "info":"status (0=good)"} + {"name":"status", "type":"B", "info":"status (0=good)"}, + {"name":"ai", "type":"B", "info":"3 bits corresponds to 3 sections; 1 = singal and 0 = noise"} ] }, { diff --git a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java index 70af041ea8..f860056984 100644 --- a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java +++ b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java @@ -104,7 +104,7 @@ class Hit { int layer; // 1 to 6 for SVT, and 7 to 12 for BMT int sector; int strip; - int index; // Index of hits in a strip in case that there are multiple hits in a strip + int index; // Index of hits in a strip, in case that there are multiple hits in a strip public Hit(int layer, int sector, int strip, int index) { this.layer = layer; @@ -140,14 +140,14 @@ public class CVTDenoiseEngine extends ReconstructionEngine { private static final int NFEATURES = 9; // Number of features for input of models // Inputs are from BST and BMT adc banks - final static String BST_BANK = "BST::adc"; - final static String BMT_BANK = "BMT::adc"; + final static String BST_BANK = "BST::Hits"; + final static String BMT_BANK = "BMT::Hits"; // Model files and thresholds for the three models; They could be set in yaml files final static String CONF_MODEL_FILES[] = {"modelFile1", "modelFile2", "modelFile3"}; final static String CONF_THRESHOLDS[] = {"threshold1", "threshold2", "threshold3"}; String[] modelFiles = {"classifier_torchscript_sector1_noCSWeight_weightInTraining.pt", "classifier_torchscript_sector2_noCSWeight_weightInTraining.pt", "classifier_torchscript_sector3_noCSWeight_weightInTraining.pt"}; - float[] thresholds = {0.025f, 0.025f, 0.025f}; // To be determined + float[] thresholds = {0.1f, 0.1f, 0.1f}; // To be determined // Number of threshold for predictor pools final static String CONF_THREADS = "threads"; @@ -245,41 +245,37 @@ public boolean processDataEvent(DataEvent event) { for(int i = 0; i < bst_bank.rows(); i++){ int sector = bst_bank.getByte("sector", i); int layer = bst_bank.getByte("layer", i); - int strip = bst_bank.getShort("component",i); - int order = bst_bank.getByte("order", i); - int adc = bst_bank.getInt("ADC", i); + int strip = bst_bank.getShort("strip",i); Hit hit = new Hit(layer, sector, strip, nHitsLayerSectorStrip_BST[layer-1][sector-1][strip-1]++); - if(order == 0 || order == 10){ - Line3D line = Geometry.getInstance().getSVT().getStrip(layer, sector, strip); - float[] features = {strip, (float)(line.origin().x()/10.), (float)(line.end().x()/10.), (float)(line.origin().y()/10.), - (float)(line.end().y()/10.), (float)(line.origin().z()/10.), (float)(line.end().z()/10.), sector, layer}; // unit conversion from mm to cm for end points - - for(int f = 0; f < NFEATURES - 1; f++){ - float min = minVals[f][layer-1]; - float max = maxVals[f][layer-1]; - features[f] = (features[f] - min) / (max-min); - } - features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); - - List sectionList = getSectionList(layer, sector); - for(int section : sectionList){ - for(int f = 0; f < NFEATURES; f++){ - x[section - 1][nHits[section - 1]][f] = features[f]; - } - - mask[section - 1][nHits[section - 1]] = 1.0f; - - maps[section - 1].put(nHits[section - 1], hit); - - nHits[section - 1]++; - if(nHits[section - 1] == MAX_HITS) { - Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); - return true; - } - } - } + Line3D line = Geometry.getInstance().getSVT().getStrip(layer, sector, strip); + float[] features = {strip, (float)(line.origin().x()/10.), (float)(line.end().x()/10.), (float)(line.origin().y()/10.), + (float)(line.end().y()/10.), (float)(line.origin().z()/10.), (float)(line.end().z()/10.), sector, layer}; // unit conversion from mm to cm for end points + + for(int f = 0; f < NFEATURES - 1; f++){ + float min = minVals[f][layer-1]; + float max = maxVals[f][layer-1]; + features[f] = (features[f] - min) / (max-min); + } + features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); + + List sectionList = getSectionList(layer, sector); + for(int section : sectionList){ + for(int f = 0; f < NFEATURES; f++){ + x[section - 1][nHits[section - 1]][f] = features[f]; + } + + mask[section - 1][nHits[section - 1]] = 1.0f; + + maps[section - 1].put(nHits[section - 1], hit); + + nHits[section - 1]++; + if(nHits[section - 1] == MAX_HITS) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); + return true; + } + } } // Read BMT bank and set input for models @@ -287,64 +283,64 @@ public boolean processDataEvent(DataEvent event) { for(int i = 0; i < bmt_bank.rows(); i++){ int sector = bmt_bank.getByte("sector", i); int layer = bmt_bank.getByte("layer", i); - int strip = bmt_bank.getShort("component",i); - int order = bmt_bank.getByte("order", i); - int adc = bst_bank.getInt("ADC", i); + int strip = bmt_bank.getShort("strip",i); Hit hit = new Hit(layer+6, sector, strip, nHitsLayerSectorStrip_BMT[layer-1][sector-1][strip-1]++); - if(order == 0 || order == 10){ - Point3D originPoint, endPoint; - int region = (layer + 1) / 2; // Get region number for a BMT layer; layers 1, 4, 6 for BMT-C layers, and layers 2, 3, 5 for BMT-Z - if(layer == 2 || layer ==3 || layer == 5) { - Line3D line = Geometry.getInstance().getBMT().getLCZstrip(region, sector, strip, swimmer); - originPoint = line.origin(); - endPoint = line.origin(); - } - else { - Arc3D arcLine = Geometry.getInstance().getBMT().getCstrip(region, sector, strip); - originPoint = arcLine.origin(); - endPoint = arcLine.end(); - } - - layer += 6; - float[] features = {strip, (float)(originPoint.x()/10.), (float)(endPoint.x()/10.), (float)(originPoint.y()/10.), - (float)(endPoint.y()/10.), (float)(originPoint.z()/10.), (float)(endPoint.z()/10.), sector, layer}; // unit conversion from mm to cm for end points - - for(int f = 0; f < NFEATURES - 1; f++){ - float min = minVals[f][layer-1]; - float max = maxVals[f][layer-1]; - features[f] = (features[f] - min) / (max-min); - } - features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); - - List sectionList = getSectionList(layer, sector); - for(int section : sectionList){ - for(int f = 0; f < NFEATURES; f++){ - x[section - 1][nHits[section - 1]][f] = features[f]; - } - - mask[section - 1][nHits[section - 1]] = 1.0f; - - maps[section - 1].put(nHits[section - 1], hit); - - nHits[section - 1]++; - if(nHits[section - 1] == MAX_HITS) { - Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); - return true; - } - } + Point3D originPoint, endPoint; + int region = (layer + 1) / 2; // Get region number for a BMT layer; layers 1, 4, 6 for BMT-C layers, and layers 2, 3, 5 for BMT-Z + if(layer == 2 || layer ==3 || layer == 5) { + Line3D line = Geometry.getInstance().getBMT().getLCZstrip(region, sector, strip, swimmer); + originPoint = line.origin(); + endPoint = line.origin(); } + else { + Arc3D arcLine = Geometry.getInstance().getBMT().getCstrip(region, sector, strip); + originPoint = arcLine.origin(); + endPoint = arcLine.end(); + } + + layer += 6; + float[] features = {strip, (float)(originPoint.x()/10.), (float)(endPoint.x()/10.), (float)(originPoint.y()/10.), + (float)(endPoint.y()/10.), (float)(originPoint.z()/10.), (float)(endPoint.z()/10.), sector, layer}; // unit conversion from mm to cm for end points + + for(int f = 0; f < NFEATURES - 1; f++){ + float min = minVals[f][layer-1]; + float max = maxVals[f][layer-1]; + features[f] = (features[f] - min) / (max-min); + } + features[NFEATURES-1] = (features[NFEATURES-1] - 1) / (NLAYERS - 1); + + List sectionList = getSectionList(layer, sector); + for(int section : sectionList){ + for(int f = 0; f < NFEATURES; f++){ + x[section - 1][nHits[section - 1]][f] = features[f]; + } + + mask[section - 1][nHits[section - 1]] = 1.0f; + + maps[section - 1].put(nHits[section - 1], hit); + + nHits[section - 1]++; + if(nHits[section - 1] == MAX_HITS) { + Logger.getLogger(CVTDenoiseEngine.class.getName()).log(Level.SEVERE, "Number of hits is over maximum limit!"); + return true; + } + } } + + // AI-based hit classification. + // The status is encoded in a byte (3-bit bitmask). + // + // Each bit corresponds to one section: + // Section 1 <-> bit 0 + // Section 2 <-> bit 1 + // Section 3 <-> bit 2 + // + // Bit value: + // 0 = noise + // 1 = signal - // Apply models for prediction of hits - // Status for a hit with order of 0: - // 0: rejected - // 1: accepted by section 1 - // 2: accepted by section 2 - // 3: accepted by section 3 - // 12: accepted by sections 1 & 2; Note: a hit is shared by secton 1 & 2, and is accpcted by both section - // 23: accepted by sections 2 & 3; Note: a hit is shared by secton 2 & 3, and is accpcted by both section int maxIndex_BST = Integer.MIN_VALUE; for (int i = 0; i < nHitsLayerSectorStrip_BST.length; i++) { for (int j = 0; j < nHitsLayerSectorStrip_BST[i].length; j++) { @@ -377,10 +373,7 @@ public boolean processDataEvent(DataEvent event) { float[][] preds = predictor.predict(input); for(int i = 0; i < nHits[s]; i++){ Hit hit = maps[s].get(i); - byte status = statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()]; - if(preds[i][0] > thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] = (byte) (status * 10 + s + 1); - //if(statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] < 0 || statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] > 123) - // System.out.println(preds[i][0] + " " + s + " " + i + " " + hit.getLayer() + " " + hit.getSector() + " " + hit.getStrip() + " " + hit.getIndex() + " " + statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()]); + if(preds[i][0] > thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << s); } } finally { predictors[s].put(predictor); @@ -482,28 +475,24 @@ private void updateBanks(DataBank bst_bank, DataBank bmt_bank, byte[][][][] stat for (int row=0; row Date: Tue, 17 Mar 2026 15:32:34 -0400 Subject: [PATCH 4/9] update byte for record of hit status by AI prediction --- etc/bankdefs/hipo4/bmt.json | 2 +- etc/bankdefs/hipo4/bst.json | 2 +- .../rec/cvt/services/CVTDenoiseEngine.java | 23 +++++++------------ 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/etc/bankdefs/hipo4/bmt.json b/etc/bankdefs/hipo4/bmt.json index 3d50008262..d46630c226 100644 --- a/etc/bankdefs/hipo4/bmt.json +++ b/etc/bankdefs/hipo4/bmt.json @@ -130,7 +130,7 @@ {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, {"name":"status", "type":"B", "info":"status (0=good)"}, - {"name":"ai", "type":"B", "info":"3 bits corresponds to 3 sections; 1 = singal and 0 = noise"} + {"name":"ai", "type":"B", "info":"bits [0-2] : noise flags for sections 1-3 (1 = noise, 0 = signal); bits [3-5] : hit presence for sections 1-3 (1 = hit present, 0 = no hit)"} ] }, { diff --git a/etc/bankdefs/hipo4/bst.json b/etc/bankdefs/hipo4/bst.json index a45c3475b1..7b8acb882c 100644 --- a/etc/bankdefs/hipo4/bst.json +++ b/etc/bankdefs/hipo4/bst.json @@ -121,7 +121,7 @@ {"name":"clusterID", "type":"S", "info":"associated cluster ID"}, {"name":"trkID", "type":"S", "info":"associated track ID"}, {"name":"status", "type":"B", "info":"status (0=good)"}, - {"name":"ai", "type":"B", "info":"3 bits corresponds to 3 sections; 1 = singal and 0 = noise"} + {"name":"ai", "type":"B", "info":"bits [0-2] : noise flags for sections 1-3 (1 = noise, 0 = signal); bits [3-5] : hit presence for sections 1-3 (1 = hit present, 0 = no hit)"} ] }, { diff --git a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java index f860056984..29030ca589 100644 --- a/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java +++ b/reconstruction/cvt/src/main/java/org/jlab/rec/cvt/services/CVTDenoiseEngine.java @@ -139,7 +139,7 @@ public class CVTDenoiseEngine extends ReconstructionEngine { private static final int MAX_HITS = 450; // Maximum of hits for each layer private static final int NFEATURES = 9; // Number of features for input of models - // Inputs are from BST and BMT adc banks + // Inputs are from BST and BMT hit banks final static String BST_BANK = "BST::Hits"; final static String BMT_BANK = "BMT::Hits"; @@ -328,19 +328,7 @@ public boolean processDataEvent(DataEvent event) { } } } - - // AI-based hit classification. - // The status is encoded in a byte (3-bit bitmask). - // - // Each bit corresponds to one section: - // Section 1 <-> bit 0 - // Section 2 <-> bit 1 - // Section 3 <-> bit 2 - // - // Bit value: - // 0 = noise - // 1 = signal - + int maxIndex_BST = Integer.MIN_VALUE; for (int i = 0; i < nHitsLayerSectorStrip_BST.length; i++) { for (int j = 0; j < nHitsLayerSectorStrip_BST[i].length; j++) { @@ -363,6 +351,10 @@ public boolean processDataEvent(DataEvent event) { } int maxIndex = Math.max(maxIndex_BST, maxIndex_BMT); + // AI-based hit classification. + // Hit status is encoded in a single byte: + // bits [0–2] : noise flags for sections 1–3 (1 = noise, 0 = signal) + // bits [3–5] : hit presence for sections 1–3 (1 = hit present, 0 = no hit) byte[][][][] statuses = new byte[NLAYERS][18][1152][maxIndex+1]; for(int s = 0; s < NSECTIONS; s++){ CVTInput input = new CVTInput(x[s],mask[s]); @@ -373,7 +365,8 @@ public boolean processDataEvent(DataEvent event) { float[][] preds = predictor.predict(input); for(int i = 0; i < nHits[s]; i++){ Hit hit = maps[s].get(i); - if(preds[i][0] > thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << s); + statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << (s+3)); + if(preds[i][0] < thresholds[s]) statuses[hit.getLayer()-1][hit.getSector()-1][hit.getStrip() - 1][hit.getIndex()] |= (byte) (1 << s); } } finally { predictors[s].put(predictor); From 4ae1f7569ac770021c07998a202d2f92df9ff324 Mon Sep 17 00:00:00 2001 From: veronique Date: Thu, 23 Apr 2026 13:51:34 -0400 Subject: [PATCH 5/9] hipo to npz converter python script to view to converted output --- .../main/java/org/jlab/io/hipo/HipoToNpz.java | 966 ++++++++++++++++++ 1 file changed, 966 insertions(+) create mode 100644 common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java new file mode 100644 index 0000000000..a77b1364df --- /dev/null +++ b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java @@ -0,0 +1,966 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package org.jlab.io.hipo; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.zip.CRC32; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.jlab.io.base.DataBank; +import org.jlab.io.base.DataEvent; +/** + * + * @author veronique + */ + +/** + * Convert selected banks from a HIPO file into a single NPZ archive. + * + * Bank selection: + * - no bank selection specified: include all banks found in the HIPO file + * - comma-separated bank names: include only those banks + * - --bank-file : one bank name per line, '#' comments allowed + * - both comma list and --bank-file may be used together + * + * Schema JSON selection: + * - --schema-dir is required + * - if banks are selected, only JSON files containing at least one selected bank are parsed + * - if no banks are selected, all JSON files in the folder are parsed + * + * Example: after compiling coatjava, cd coatjava, run the script: + * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json + * + * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json BST::adc,RUN::config + * + * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json --bank-file banks.txt + */ +public class HipoToNpz { + + public static void main(String[] args) throws Exception { + CliOptions options = CliOptions.parse(args); + + Map schemaIndex = SchemaLoader.indexSchemaFiles(options.schemaDir); + Set schemaFilesToLoad = SchemaLoader.chooseSchemaFiles(schemaIndex, options.selectedBanks); + + if (schemaFilesToLoad.isEmpty()) { + throw new IllegalStateException("No schema JSON files selected from " + options.schemaDir.getAbsolutePath()); + } + + Map schemaTypes = SchemaLoader.loadSchemaTypes(schemaFilesToLoad); + + System.out.println("Schema dir : " + options.schemaDir.getAbsolutePath()); + System.out.println("Schema files : " + schemaFilesToLoad.size()); + if (options.selectedBanks == null) { + System.out.println("Selected banks : ALL"); + } else { + System.out.println("Selected banks : " + options.selectedBanks.size()); + for (String b : options.selectedBanks) { + System.out.println(" " + b); + } + } + + Converter converter = new Converter(options.selectedBanks, schemaTypes); + converter.convert(options.input, options.output); + } + + // ------------------------------------------------------------------------ + // CLI + // ------------------------------------------------------------------------ + + private static final class CliOptions { + final File input; + final File output; + final File schemaDir; + final Set selectedBanks; // null means all banks + + private CliOptions(File input, File output, File schemaDir, Set selectedBanks) { + this.input = input; + this.output = output; + this.schemaDir = schemaDir; + this.selectedBanks = selectedBanks; + } + + static CliOptions parse(String[] args) throws Exception { + if (args.length < 4) { + printUsageAndExit(); + } + + File input = new File(args[0]); + File output = new File(args[1]); + + File schemaDir = null; + Set selectedBanks = new LinkedHashSet<>(); + boolean selectAll = true; + + for (int i = 2; i < args.length; i++) { + String arg = args[i]; + if (arg == null || arg.isBlank()) { + continue; + } + + if ("--schema-dir".equals(arg)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--schema-dir requires a directory path"); + } + schemaDir = new File(args[++i]); + continue; + } + + if ("--bank-file".equals(arg)) { + if (i + 1 >= args.length) { + throw new IllegalArgumentException("--bank-file requires a file path"); + } + File bankFile = new File(args[++i]); + selectedBanks.addAll(readBankNames(bankFile)); + selectAll = false; + continue; + } + + if ("*".equals(arg)) { + selectedBanks.clear(); + selectAll = true; + continue; + } + + for (String s : arg.split(",")) { + String bank = s.trim(); + if (!bank.isEmpty()) { + selectedBanks.add(bank); + selectAll = false; + } + } + } + + if (schemaDir == null) { + throw new IllegalArgumentException("--schema-dir is required"); + } + if (!schemaDir.isDirectory()) { + throw new IllegalArgumentException("Schema directory not found: " + schemaDir.getAbsolutePath()); + } + if (!input.exists()) { + throw new IllegalArgumentException("Input HIPO file not found: " + input.getAbsolutePath()); + } + + return new CliOptions(input, output, schemaDir, selectAll ? null : selectedBanks); + } + + private static Set readBankNames(File bankFile) throws IOException { + if (!bankFile.exists()) { + throw new IOException("Bank file not found: " + bankFile.getAbsolutePath()); + } + + Set names = new LinkedHashSet<>(); + for (String line : Files.readAllLines(bankFile.toPath(), StandardCharsets.UTF_8)) { + String s = line.trim(); + if (s.isEmpty() || s.startsWith("#")) { + continue; + } + names.add(s); + } + return names; + } + + private static void printUsageAndExit() { + System.err.println("Usage:"); + System.err.println(" java org.jlab.io.hipo.HipoToNpz --schema-dir [bank1,bank2,...|*] [--bank-file banks.txt]"); + System.exit(2); + } + } + + // ------------------------------------------------------------------------ + // Types + // ------------------------------------------------------------------------ + + private enum ColumnType { + BYTE(" BYTE; + case "S" -> SHORT; + case "I" -> INT; + case "L" -> LONG; + case "F" -> FLOAT; + case "D" -> DOUBLE; + default -> throw new IllegalStateException("Unknown schema type '" + code + "' for " + fullName); + }; + } + } + + // ------------------------------------------------------------------------ + // Converter + // ------------------------------------------------------------------------ + + private static final class Converter { + private final Map banks = new LinkedHashMap<>(); + private final Set selectedBanks; // null means all banks + private final Map schemaTypes; // BANK/COLUMN -> type + + Converter(Set selectedBanks, Map schemaTypes) { + this.selectedBanks = selectedBanks; + this.schemaTypes = schemaTypes; + } + + void convert(File input, File output) throws Exception { + if (selectedBanks == null) { + System.out.println("Including all banks"); + } else { + System.out.println("Including selected banks only"); + } + + HipoDataSource reader = new HipoDataSource(); + reader.open(input); + + long nEvents = 0; + while (reader.hasEvent()) { + DataEvent event = reader.getNextEvent(); + nEvents++; + ingestEvent(event); + + if ((nEvents % 10000) == 0) { + System.out.printf("Processed %,d events%n", nEvents); + } + } + reader.close(); + + writeNpz(output); + System.out.printf("Wrote %s with %,d events and %,d banks%n", + output.getAbsolutePath(), nEvents, banks.size()); + } + + private boolean keepBank(String bankName) { + return selectedBanks == null || selectedBanks.contains(bankName); + } + + private void ingestEvent(DataEvent event) { + String[] bankNames = event.getBankList(); + if (bankNames == null) { + return; + } + + Map presentRows = new HashMap<>(); + + for (String bankName : bankNames) { + if (bankName == null || bankName.isBlank()) { + continue; + } + if (!keepBank(bankName)) { + continue; + } + + DataBank bank = event.getBank(bankName); + if (bank == null) { + continue; + } + + int rows = bank.rows(); + presentRows.put(bankName, rows); + + BankStore store = banks.computeIfAbsent(bankName, BankStore::new); + ensureColumns(store, bank); + + String[] cols = bank.getColumnList(); + if (cols != null) { + for (String col : cols) { + ColumnStore cstore = store.columns.get(col); + if (cstore == null) { + continue; + } + for (int r = 0; r < rows; r++) { + cstore.append(bank, col, r); + } + } + } + } + + for (BankStore store : banks.values()) { + int rows = presentRows.getOrDefault(store.bankName, 0); + store.appendEventRows(rows); + } + } + + private void ensureColumns(BankStore store, DataBank bank) { + String[] cols = bank.getColumnList(); + if (cols == null) { + return; + } + + for (String col : cols) { + if (store.columns.containsKey(col)) { + continue; + } + ColumnType type = discoverColumnType(bank, col); + store.columns.put(col, new ColumnStore(store.bankName, col, type)); + } + } + + private ColumnType discoverColumnType(DataBank bank, String col) { + String fullName = bank.getDescriptor().getName() + "/" + col; + + ColumnType schemaType = schemaTypes.get(fullName); + if (schemaType != null) { + return schemaType; + } + + // Fallback to descriptor only if schema map lacks the bank/column. + Object desc = bank.getDescriptor(); + if (desc == null) { + throw new IllegalStateException("No descriptor and no schema entry for " + fullName); + } + + String typeName = tryGetTypeName(desc, col); + if (typeName != null) { + String s = typeName.trim(); + if (!s.isEmpty() && !s.equalsIgnoreCase("undefined")) { + return mapTypeName(s, fullName); + } + } + + Integer typeCode = tryGetTypeCode(desc, col); + if (typeCode != null) { + return mapTypeCode(typeCode, fullName); + } + + throw new IllegalStateException("Could not determine type for " + fullName); + } + + private Integer tryGetTypeCode(Object desc, String col) { + try { + Object v = desc.getClass() + .getMethod("getProperty", String.class, String.class) + .invoke(desc, "type", col); + if (v instanceof Number n) { + return n.intValue(); + } + } catch (ReflectiveOperationException ignored) { + } + + try { + Object v = desc.getClass() + .getMethod("getProperty", String.class) + .invoke(desc, col + ".type"); + if (v instanceof Number n) { + return n.intValue(); + } + } catch (ReflectiveOperationException ignored) { + } + + return null; + } + + private String tryGetTypeName(Object desc, String col) { + try { + Object v = desc.getClass() + .getMethod("getPropertyString", String.class, String.class) + .invoke(desc, "type", col); + if (v != null) { + return v.toString(); + } + } catch (ReflectiveOperationException ignored) { + } + + try { + Object v = desc.getClass() + .getMethod("getPropertyString", String.class) + .invoke(desc, col + ".type"); + if (v != null) { + return v.toString(); + } + } catch (ReflectiveOperationException ignored) { + } + + return null; + } + + private ColumnType mapTypeCode(int t, String fullName) { + return switch (t) { + case 1 -> ColumnType.INT; + case 2 -> ColumnType.FLOAT; + case 3 -> ColumnType.DOUBLE; + case 4 -> ColumnType.SHORT; + case 5 -> ColumnType.BYTE; + case 6 -> ColumnType.LONG; + case 8 -> ColumnType.BYTE; + default -> throw new IllegalStateException("Unknown type code " + t + " for " + fullName); + }; + } + + private ColumnType mapTypeName(String typeName, String fullName) { + String s = typeName.trim().toLowerCase(); + return switch (s) { + case "int", "int32", "i4" -> ColumnType.INT; + case "float", "float32", "f4" -> ColumnType.FLOAT; + case "double", "float64", "f8" -> ColumnType.DOUBLE; + case "short", "int16", "i2" -> ColumnType.SHORT; + case "byte", "int8", "i1" -> ColumnType.BYTE; + case "long", "int64", "i8" -> ColumnType.LONG; + default -> throw new IllegalStateException("Unknown type name '" + typeName + "' for " + fullName); + }; + } + + private void writeNpz(File output) throws IOException { + try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(output)))) { + zos.setLevel(Deflater.BEST_SPEED); + + for (BankStore bank : banks.values()) { + String bankBase = sanitize(bank.bankName); + + addEntry(zos, bankBase + "__rows_per_event.npy", + Npy.writeIntArray(bank.rowsPerEvent.toArray(), " indexSchemaFiles(File schemaDir) throws IOException { + File[] files = schemaDir.listFiles((dir, name) -> name.toLowerCase().endsWith(".json")); + if (files == null || files.length == 0) { + throw new IOException("No .json schema files found in " + schemaDir.getAbsolutePath()); + } + + Map bankToFile = new HashMap<>(); + for (File file : files) { + Object root = Json.parse(Files.readString(file.toPath(), StandardCharsets.UTF_8)); + if (!(root instanceof List banks)) { + continue; + } + + for (Object bankObj : banks) { + if (!(bankObj instanceof Map bankMap)) { + continue; + } + Object nameObj = bankMap.get("name"); + if (nameObj instanceof String bankName && !bankName.isBlank()) { + bankToFile.put(bankName, file); + } + } + } + return bankToFile; + } + + static Set chooseSchemaFiles(Map bankToFile, Set selectedBanks) { + Set files = new LinkedHashSet<>(); + + if (selectedBanks == null) { + files.addAll(bankToFile.values()); + return files; + } + + List missing = new ArrayList<>(); + for (String bank : selectedBanks) { + File file = bankToFile.get(bank); + if (file != null) { + files.add(file); + } else { + missing.add(bank); + } + } + + if (!missing.isEmpty()) { + System.err.println("Warning: no schema JSON file found for selected banks:"); + for (String b : missing) { + System.err.println(" " + b); + } + } + + return files; + } + + static Map loadSchemaTypes(Set schemaFiles) throws IOException { + Map types = new HashMap<>(); + + List sorted = new ArrayList<>(schemaFiles); + sorted.sort(Comparator.comparing(File::getName)); + + for (File file : sorted) { + Object root = Json.parse(Files.readString(file.toPath(), StandardCharsets.UTF_8)); + if (!(root instanceof List banks)) { + continue; + } + + for (Object bankObj : banks) { + if (!(bankObj instanceof Map bankMap)) { + continue; + } + + Object nameObj = bankMap.get("name"); + Object entriesObj = bankMap.get("entries"); + if (!(nameObj instanceof String bankName) || !(entriesObj instanceof List entries)) { + continue; + } + + for (Object entryObj : entries) { + if (!(entryObj instanceof Map entryMap)) { + continue; + } + + Object entryNameObj = entryMap.get("name"); + Object entryTypeObj = entryMap.get("type"); + if (!(entryNameObj instanceof String colName) || !(entryTypeObj instanceof String typeCode)) { + continue; + } + + String fullName = bankName + "/" + colName; + types.put(fullName, ColumnType.fromSchemaCode(typeCode, fullName)); + } + } + } + + return types; + } + } + + // ------------------------------------------------------------------------ + // Stores + // ------------------------------------------------------------------------ + + private static final class BankStore { + final String bankName; + final Map columns = new LinkedHashMap<>(); + final IntList rowsPerEvent = new IntList(); + final LongList offsets = new LongList(); + long totalRows = 0; + + BankStore(String bankName) { + this.bankName = bankName; + this.offsets.add(0L); + } + + void appendEventRows(int rows) { + rowsPerEvent.add(rows); + totalRows += rows; + offsets.add(totalRows); + } + } + + private static final class ColumnStore { + final String bankName; + final String columnName; + final ColumnType type; + final ByteList bytes; + final ShortList shorts; + final IntList ints; + final LongList longs; + final FloatList floats; + final DoubleList doubles; + + ColumnStore(String bankName, String columnName, ColumnType type) { + this.bankName = bankName; + this.columnName = columnName; + this.type = type; + this.bytes = type == ColumnType.BYTE ? new ByteList() : null; + this.shorts = type == ColumnType.SHORT ? new ShortList() : null; + this.ints = type == ColumnType.INT ? new IntList() : null; + this.longs = type == ColumnType.LONG ? new LongList() : null; + this.floats = type == ColumnType.FLOAT ? new FloatList() : null; + this.doubles = type == ColumnType.DOUBLE ? new DoubleList() : null; + } + + void append(DataBank bank, String col, int row) { + switch (type) { + case BYTE -> bytes.add(bank.getByte(col, row)); + case SHORT -> shorts.add(bank.getShort(col, row)); + case INT -> ints.add(bank.getInt(col, row)); + case LONG -> longs.add(bank.getLong(col, row)); + case FLOAT -> floats.add(bank.getFloat(col, row)); + case DOUBLE -> doubles.add(bank.getDouble(col, row)); + } + } + + byte[] toNpyBytes() throws IOException { + return switch (type) { + case BYTE -> Npy.writeByteArray(bytes.toArray(), type.npyDescr); + case SHORT -> Npy.writeShortArray(shorts.toArray(), type.npyDescr); + case INT -> Npy.writeIntArray(ints.toArray(), type.npyDescr); + case LONG -> Npy.writeLongArray(longs.toArray(), type.npyDescr); + case FLOAT -> Npy.writeFloatArray(floats.toArray(), type.npyDescr); + case DOUBLE -> Npy.writeDoubleArray(doubles.toArray(), type.npyDescr); + }; + } + } + + // ------------------------------------------------------------------------ + // Minimal JSON parser + // ------------------------------------------------------------------------ + + private static final class Json { + static Object parse(String text) { + return new Parser(text).parseValue(); + } + + private static final class Parser { + private final String s; + private int i = 0; + + Parser(String s) { + this.s = s; + } + + Object parseValue() { + skipWs(); + if (i >= s.length()) { + throw new IllegalStateException("Unexpected end of JSON"); + } + + char c = s.charAt(i); + return switch (c) { + case '{' -> parseObject(); + case '[' -> parseArray(); + case '"' -> parseString(); + case 't' -> parseTrue(); + case 'f' -> parseFalse(); + case 'n' -> parseNull(); + default -> parseNumber(); + }; + } + + private Map parseObject() { + expect('{'); + Map obj = new LinkedHashMap<>(); + skipWs(); + if (peek('}')) { + expect('}'); + return obj; + } + + while (true) { + skipWs(); + String key = parseString(); + skipWs(); + expect(':'); + Object value = parseValue(); + obj.put(key, value); + skipWs(); + if (peek('}')) { + expect('}'); + break; + } + expect(','); + } + return obj; + } + + private List parseArray() { + expect('['); + List arr = new ArrayList<>(); + skipWs(); + if (peek(']')) { + expect(']'); + return arr; + } + + while (true) { + arr.add(parseValue()); + skipWs(); + if (peek(']')) { + expect(']'); + break; + } + expect(','); + } + return arr; + } + + private String parseString() { + expect('"'); + StringBuilder sb = new StringBuilder(); + while (i < s.length()) { + char c = s.charAt(i++); + if (c == '"') { + return sb.toString(); + } + if (c == '\\') { + if (i >= s.length()) { + throw new IllegalStateException("Bad escape"); + } + char e = s.charAt(i++); + switch (e) { + case '"', '\\', '/' -> sb.append(e); + case 'b' -> sb.append('\b'); + case 'f' -> sb.append('\f'); + case 'n' -> sb.append('\n'); + case 'r' -> sb.append('\r'); + case 't' -> sb.append('\t'); + case 'u' -> { + if (i + 4 > s.length()) { + throw new IllegalStateException("Bad unicode escape"); + } + String hex = s.substring(i, i + 4); + sb.append((char) Integer.parseInt(hex, 16)); + i += 4; + } + default -> throw new IllegalStateException("Bad escape: \\" + e); + } + } else { + sb.append(c); + } + } + throw new IllegalStateException("Unterminated string"); + } + + private Object parseNumber() { + int start = i; + if (s.charAt(i) == '-') { + i++; + } + while (i < s.length() && Character.isDigit(s.charAt(i))) { + i++; + } + boolean isFloat = false; + if (i < s.length() && s.charAt(i) == '.') { + isFloat = true; + i++; + while (i < s.length() && Character.isDigit(s.charAt(i))) { + i++; + } + } + if (i < s.length() && (s.charAt(i) == 'e' || s.charAt(i) == 'E')) { + isFloat = true; + i++; + if (i < s.length() && (s.charAt(i) == '+' || s.charAt(i) == '-')) { + i++; + } + while (i < s.length() && Character.isDigit(s.charAt(i))) { + i++; + } + } + + String num = s.substring(start, i); + return isFloat ? Double.parseDouble(num) : Long.parseLong(num); + } + + private Boolean parseTrue() { + expect('t'); expect('r'); expect('u'); expect('e'); + return Boolean.TRUE; + } + + private Boolean parseFalse() { + expect('f'); expect('a'); expect('l'); expect('s'); expect('e'); + return Boolean.FALSE; + } + + private Object parseNull() { + expect('n'); expect('u'); expect('l'); expect('l'); + return null; + } + + private void skipWs() { + while (i < s.length()) { + char c = s.charAt(i); + if (c == ' ' || c == '\n' || c == '\r' || c == '\t') { + i++; + } else { + break; + } + } + } + + private boolean peek(char c) { + skipWs(); + return i < s.length() && s.charAt(i) == c; + } + + private void expect(char c) { + skipWs(); + if (i >= s.length() || s.charAt(i) != c) { + throw new IllegalStateException("Expected '" + c + "' at position " + i); + } + i++; + } + } + } + + // ------------------------------------------------------------------------ + // NPY writer + // ------------------------------------------------------------------------ + + private static final class Npy { + private static final byte[] MAGIC = {(byte) 0x93, 'N', 'U', 'M', 'P', 'Y'}; + + static byte[] writeByteArray(byte[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + out.write(values); + return out.toByteArray(); + } + + static byte[] writeShortArray(short[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 2).order(ByteOrder.LITTLE_ENDIAN); + for (short v : values) bb.putShort(v); + out.write(bb.array()); + return out.toByteArray(); + } + + static byte[] writeIntArray(int[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); + for (int v : values) bb.putInt(v); + out.write(bb.array()); + return out.toByteArray(); + } + + static byte[] writeLongArray(long[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); + for (long v : values) bb.putLong(v); + out.write(bb.array()); + return out.toByteArray(); + } + + static byte[] writeFloatArray(float[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); + for (float v : values) bb.putFloat(v); + out.write(bb.array()); + return out.toByteArray(); + } + + static byte[] writeDoubleArray(double[] values, String descr) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writeHeader(out, descr, values.length); + ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); + for (double v : values) bb.putDouble(v); + out.write(bb.array()); + return out.toByteArray(); + } + + private static void writeHeader(ByteArrayOutputStream out, String descr, int length) throws IOException { + out.write(MAGIC); + out.write(1); + out.write(0); + + String dict = "{'descr': '" + descr + "', 'fortran_order': False, 'shape': (" + length + ",), }"; + int preamble = MAGIC.length + 2 + 2; + int padLen = 16 - ((preamble + dict.length() + 1) % 16); + if (padLen == 16) { + padLen = 0; + } + + String fullHeader = dict + " ".repeat(padLen) + "\n"; + byte[] fullHeaderBytes = fullHeader.getBytes(StandardCharsets.US_ASCII); + + ByteBuffer hlen = ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN); + hlen.putShort((short) fullHeaderBytes.length); + out.write(hlen.array()); + out.write(fullHeaderBytes); + } + } + + // ------------------------------------------------------------------------ + // Primitive dynamic arrays + // ------------------------------------------------------------------------ + + private static final class ByteList { + private byte[] data = new byte[1024]; + private int size = 0; + void add(byte v) { ensure(size + 1); data[size++] = v; } + byte[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class ShortList { + private short[] data = new short[1024]; + private int size = 0; + void add(short v) { ensure(size + 1); data[size++] = v; } + short[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class IntList { + private int[] data = new int[1024]; + private int size = 0; + void add(int v) { ensure(size + 1); data[size++] = v; } + int[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class LongList { + private long[] data = new long[1024]; + private int size = 0; + void add(long v) { ensure(size + 1); data[size++] = v; } + long[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class FloatList { + private float[] data = new float[1024]; + private int size = 0; + void add(float v) { ensure(size + 1); data[size++] = v; } + float[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } + + private static final class DoubleList { + private double[] data = new double[1024]; + private int size = 0; + void add(double v) { ensure(size + 1); data[size++] = v; } + double[] toArray() { return Arrays.copyOf(data, size); } + private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } + } +} \ No newline at end of file From bd0846bd3e0e64d49c32e70ffd41c0e2087e9e54 Mon Sep 17 00:00:00 2001 From: veronique Date: Thu, 23 Apr 2026 14:02:13 -0400 Subject: [PATCH 6/9] hipo to npz converter python script to view to converted output --- HipoToNpz_README | 10 +++ bin/hipoToNpz | 10 +++ list_npz_banks.py | 166 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 HipoToNpz_README create mode 100755 bin/hipoToNpz create mode 100644 list_npz_banks.py diff --git a/HipoToNpz_README b/HipoToNpz_README new file mode 100644 index 0000000000..8f6ac0517f --- /dev/null +++ b/HipoToNpz_README @@ -0,0 +1,10 @@ +To convert a hipo file to npz format for a given set of banks: after compiling coatjava, cd coatjava, run the script: + * for all available banks: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 + * for BST::adc,RUN::config banks: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 BST::adc,RUN::config + * for banks listed in banks.txt: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 --bank-file banks.txt + +To inspect the npz file for n events: + * python3 list_npz_banks.py /pathTo/MyFile.npz n +Or + * python list_npz_banks.py /pathTo/MyFile.npz n + diff --git a/bin/hipoToNpz b/bin/hipoToNpz new file mode 100755 index 0000000000..da090d90af --- /dev/null +++ b/bin/hipoToNpz @@ -0,0 +1,10 @@ +#!/bin/bash + +. `dirname $0`/../libexec/env.sh + +export MALLOC_ARENA_MAX=1 + +java ${JAVA_OPTS-} -Xmx1536m -Xms1024m -XX:+UseSerialGC \ + -cp ${COATJAVA_CLASSPATH:-''} \ + org.jlab.io.hipo.HipoToNpz \ + $* diff --git a/list_npz_banks.py b/list_npz_banks.py new file mode 100644 index 0000000000..930e930a9d --- /dev/null +++ b/list_npz_banks.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +Show values for a few events from all banks in an NPZ produced by HipoToNpz. + +Usage: + python list_npz_banks.py file.npz + python list_npz_banks.py file.npz 3 +""" + +from __future__ import annotations + +import ast +import struct +import sys +from collections import defaultdict +from pathlib import Path +from zipfile import ZipFile + + +def read_npy_from_bytes(blob: bytes): + if len(blob) < 10 or blob[:6] != b"\x93NUMPY": + raise ValueError("Not a valid .npy payload") + + major = blob[6] + minor = blob[7] + + if major == 1: + header_len = struct.unpack(" 3: + print("Usage: python3 show_all_npz_events_no_numpy.py [N_EVENTS]") + sys.exit(2) + + npz_path = Path(sys.argv[1]) + n_events = int(sys.argv[2]) if len(sys.argv) == 3 else 3 + + if not npz_path.exists(): + print(f"File not found: {npz_path}") + sys.exit(1) + + arrays = load_npz(npz_path) + banks = collect_banks(arrays.keys()) + + for bank in banks: + rows_key = f"{bank}__rows_per_event" + offsets_key = f"{bank}__offsets" + + if rows_key not in arrays or offsets_key not in arrays: + continue + + rows_per_event = arrays[rows_key] + offsets = arrays[offsets_key] + + column_keys = sorted( + k for k in arrays.keys() + if k.startswith(bank + "__") and k not in (rows_key, offsets_key) + ) + + print("=" * 80) + print(f"BANK: {bank}") + print("COLUMNS:") + for key in column_keys: + print(f" {key[len(bank) + 2:]}") + print() + + nevt = min(n_events, len(rows_per_event)) + + for evt in range(nevt): + start = int(offsets[evt]) + stop = int(offsets[evt + 1]) + nrows = int(rows_per_event[evt]) + + print(f"Event {evt}: rows={nrows} slice=[{start}:{stop}]") + if nrows == 0: + print(" ") + print() + continue + + for key in column_keys: + col = key[len(bank) + 2:] + values = arrays[key][start:stop] + print(f" {col}: {list(values)}") + print() + + +if __name__ == "__main__": + main() From e6ba6f5854fddbf57436ae4631712a25cba2bd3d Mon Sep 17 00:00:00 2001 From: veronique Date: Thu, 23 Apr 2026 14:02:59 -0400 Subject: [PATCH 7/9] yaml file to create FML bank --- etc/services/data-cvthits.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 etc/services/data-cvthits.yaml diff --git a/etc/services/data-cvthits.yaml b/etc/services/data-cvthits.yaml new file mode 100644 index 0000000000..90d59aaa45 --- /dev/null +++ b/etc/services/data-cvthits.yaml @@ -0,0 +1,30 @@ +io-services: + reader: + class: org.jlab.io.clara.HipoToHipoReader + name: HipoToHipoReader + writer: + class: org.jlab.io.clara.HipoToHipoWriter + name: HipoToHipoWriter +services: + - class: org.jlab.clas.swimtools.MagFieldsEngine + name: MAGFIELDS + - class: org.jlab.rec.cvt.services.CVTHitEngine + name: CVTH +configuration: + global: + variation: rga_fall2018_bg +# timestamp: 12/31/2020-00:00:00 +# triggerMask: "0x1" +## uncomment the following two lines for compatibility with alignments before the DC fixes +# dcMinistagger: "NOTONREFWIRE" +# dcFeedthroughs: "OFF" +# io-services: +# writer: +# schema_dir: dst + services: + MAGFIELDS: + magfieldSolenoidMap: Symm_solenoid_r601_phi1_z1201_13June2018.dat + magfieldTorusMap: Full_torus_r251_phi181_z251_25Jan2021.dat + +mime-types: + - binary/data-hipo From 44ffd4ab23ac0767ab4f8d32e7e5c07ec3a3b216 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 27 Jul 2026 16:14:46 -0400 Subject: [PATCH 8/9] refactor!: remove NPZ conversion stuff from this branch --- HipoToNpz_README | 10 - bin/hipoToNpz | 10 - .../main/java/org/jlab/io/hipo/HipoToNpz.java | 966 ------------------ list_npz_banks.py | 166 --- 4 files changed, 1152 deletions(-) delete mode 100644 HipoToNpz_README delete mode 100755 bin/hipoToNpz delete mode 100644 common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java delete mode 100644 list_npz_banks.py diff --git a/HipoToNpz_README b/HipoToNpz_README deleted file mode 100644 index 8f6ac0517f..0000000000 --- a/HipoToNpz_README +++ /dev/null @@ -1,10 +0,0 @@ -To convert a hipo file to npz format for a given set of banks: after compiling coatjava, cd coatjava, run the script: - * for all available banks: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 - * for BST::adc,RUN::config banks: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 BST::adc,RUN::config - * for banks listed in banks.txt: ./bin/hipoToNpz input.hipo output.npz --schema-dir etc/bankdefs/hipo4 --bank-file banks.txt - -To inspect the npz file for n events: - * python3 list_npz_banks.py /pathTo/MyFile.npz n -Or - * python list_npz_banks.py /pathTo/MyFile.npz n - diff --git a/bin/hipoToNpz b/bin/hipoToNpz deleted file mode 100755 index da090d90af..0000000000 --- a/bin/hipoToNpz +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -. `dirname $0`/../libexec/env.sh - -export MALLOC_ARENA_MAX=1 - -java ${JAVA_OPTS-} -Xmx1536m -Xms1024m -XX:+UseSerialGC \ - -cp ${COATJAVA_CLASSPATH:-''} \ - org.jlab.io.hipo.HipoToNpz \ - $* diff --git a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java b/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java deleted file mode 100644 index a77b1364df..0000000000 --- a/common-tools/clas-io/src/main/java/org/jlab/io/hipo/HipoToNpz.java +++ /dev/null @@ -1,966 +0,0 @@ -/* - * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license - * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template - */ -package org.jlab.io.hipo; - -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.zip.CRC32; -import java.util.zip.Deflater; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import org.jlab.io.base.DataBank; -import org.jlab.io.base.DataEvent; -/** - * - * @author veronique - */ - -/** - * Convert selected banks from a HIPO file into a single NPZ archive. - * - * Bank selection: - * - no bank selection specified: include all banks found in the HIPO file - * - comma-separated bank names: include only those banks - * - --bank-file : one bank name per line, '#' comments allowed - * - both comma list and --bank-file may be used together - * - * Schema JSON selection: - * - --schema-dir is required - * - if banks are selected, only JSON files containing at least one selected bank are parsed - * - if no banks are selected, all JSON files in the folder are parsed - * - * Example: after compiling coatjava, cd coatjava, run the script: - * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json - * - * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json BST::adc,RUN::config - * - * ./bin/hipoToNpz input.hipo output.npz --schema-dir /path/to/json --bank-file banks.txt - */ -public class HipoToNpz { - - public static void main(String[] args) throws Exception { - CliOptions options = CliOptions.parse(args); - - Map schemaIndex = SchemaLoader.indexSchemaFiles(options.schemaDir); - Set schemaFilesToLoad = SchemaLoader.chooseSchemaFiles(schemaIndex, options.selectedBanks); - - if (schemaFilesToLoad.isEmpty()) { - throw new IllegalStateException("No schema JSON files selected from " + options.schemaDir.getAbsolutePath()); - } - - Map schemaTypes = SchemaLoader.loadSchemaTypes(schemaFilesToLoad); - - System.out.println("Schema dir : " + options.schemaDir.getAbsolutePath()); - System.out.println("Schema files : " + schemaFilesToLoad.size()); - if (options.selectedBanks == null) { - System.out.println("Selected banks : ALL"); - } else { - System.out.println("Selected banks : " + options.selectedBanks.size()); - for (String b : options.selectedBanks) { - System.out.println(" " + b); - } - } - - Converter converter = new Converter(options.selectedBanks, schemaTypes); - converter.convert(options.input, options.output); - } - - // ------------------------------------------------------------------------ - // CLI - // ------------------------------------------------------------------------ - - private static final class CliOptions { - final File input; - final File output; - final File schemaDir; - final Set selectedBanks; // null means all banks - - private CliOptions(File input, File output, File schemaDir, Set selectedBanks) { - this.input = input; - this.output = output; - this.schemaDir = schemaDir; - this.selectedBanks = selectedBanks; - } - - static CliOptions parse(String[] args) throws Exception { - if (args.length < 4) { - printUsageAndExit(); - } - - File input = new File(args[0]); - File output = new File(args[1]); - - File schemaDir = null; - Set selectedBanks = new LinkedHashSet<>(); - boolean selectAll = true; - - for (int i = 2; i < args.length; i++) { - String arg = args[i]; - if (arg == null || arg.isBlank()) { - continue; - } - - if ("--schema-dir".equals(arg)) { - if (i + 1 >= args.length) { - throw new IllegalArgumentException("--schema-dir requires a directory path"); - } - schemaDir = new File(args[++i]); - continue; - } - - if ("--bank-file".equals(arg)) { - if (i + 1 >= args.length) { - throw new IllegalArgumentException("--bank-file requires a file path"); - } - File bankFile = new File(args[++i]); - selectedBanks.addAll(readBankNames(bankFile)); - selectAll = false; - continue; - } - - if ("*".equals(arg)) { - selectedBanks.clear(); - selectAll = true; - continue; - } - - for (String s : arg.split(",")) { - String bank = s.trim(); - if (!bank.isEmpty()) { - selectedBanks.add(bank); - selectAll = false; - } - } - } - - if (schemaDir == null) { - throw new IllegalArgumentException("--schema-dir is required"); - } - if (!schemaDir.isDirectory()) { - throw new IllegalArgumentException("Schema directory not found: " + schemaDir.getAbsolutePath()); - } - if (!input.exists()) { - throw new IllegalArgumentException("Input HIPO file not found: " + input.getAbsolutePath()); - } - - return new CliOptions(input, output, schemaDir, selectAll ? null : selectedBanks); - } - - private static Set readBankNames(File bankFile) throws IOException { - if (!bankFile.exists()) { - throw new IOException("Bank file not found: " + bankFile.getAbsolutePath()); - } - - Set names = new LinkedHashSet<>(); - for (String line : Files.readAllLines(bankFile.toPath(), StandardCharsets.UTF_8)) { - String s = line.trim(); - if (s.isEmpty() || s.startsWith("#")) { - continue; - } - names.add(s); - } - return names; - } - - private static void printUsageAndExit() { - System.err.println("Usage:"); - System.err.println(" java org.jlab.io.hipo.HipoToNpz --schema-dir [bank1,bank2,...|*] [--bank-file banks.txt]"); - System.exit(2); - } - } - - // ------------------------------------------------------------------------ - // Types - // ------------------------------------------------------------------------ - - private enum ColumnType { - BYTE(" BYTE; - case "S" -> SHORT; - case "I" -> INT; - case "L" -> LONG; - case "F" -> FLOAT; - case "D" -> DOUBLE; - default -> throw new IllegalStateException("Unknown schema type '" + code + "' for " + fullName); - }; - } - } - - // ------------------------------------------------------------------------ - // Converter - // ------------------------------------------------------------------------ - - private static final class Converter { - private final Map banks = new LinkedHashMap<>(); - private final Set selectedBanks; // null means all banks - private final Map schemaTypes; // BANK/COLUMN -> type - - Converter(Set selectedBanks, Map schemaTypes) { - this.selectedBanks = selectedBanks; - this.schemaTypes = schemaTypes; - } - - void convert(File input, File output) throws Exception { - if (selectedBanks == null) { - System.out.println("Including all banks"); - } else { - System.out.println("Including selected banks only"); - } - - HipoDataSource reader = new HipoDataSource(); - reader.open(input); - - long nEvents = 0; - while (reader.hasEvent()) { - DataEvent event = reader.getNextEvent(); - nEvents++; - ingestEvent(event); - - if ((nEvents % 10000) == 0) { - System.out.printf("Processed %,d events%n", nEvents); - } - } - reader.close(); - - writeNpz(output); - System.out.printf("Wrote %s with %,d events and %,d banks%n", - output.getAbsolutePath(), nEvents, banks.size()); - } - - private boolean keepBank(String bankName) { - return selectedBanks == null || selectedBanks.contains(bankName); - } - - private void ingestEvent(DataEvent event) { - String[] bankNames = event.getBankList(); - if (bankNames == null) { - return; - } - - Map presentRows = new HashMap<>(); - - for (String bankName : bankNames) { - if (bankName == null || bankName.isBlank()) { - continue; - } - if (!keepBank(bankName)) { - continue; - } - - DataBank bank = event.getBank(bankName); - if (bank == null) { - continue; - } - - int rows = bank.rows(); - presentRows.put(bankName, rows); - - BankStore store = banks.computeIfAbsent(bankName, BankStore::new); - ensureColumns(store, bank); - - String[] cols = bank.getColumnList(); - if (cols != null) { - for (String col : cols) { - ColumnStore cstore = store.columns.get(col); - if (cstore == null) { - continue; - } - for (int r = 0; r < rows; r++) { - cstore.append(bank, col, r); - } - } - } - } - - for (BankStore store : banks.values()) { - int rows = presentRows.getOrDefault(store.bankName, 0); - store.appendEventRows(rows); - } - } - - private void ensureColumns(BankStore store, DataBank bank) { - String[] cols = bank.getColumnList(); - if (cols == null) { - return; - } - - for (String col : cols) { - if (store.columns.containsKey(col)) { - continue; - } - ColumnType type = discoverColumnType(bank, col); - store.columns.put(col, new ColumnStore(store.bankName, col, type)); - } - } - - private ColumnType discoverColumnType(DataBank bank, String col) { - String fullName = bank.getDescriptor().getName() + "/" + col; - - ColumnType schemaType = schemaTypes.get(fullName); - if (schemaType != null) { - return schemaType; - } - - // Fallback to descriptor only if schema map lacks the bank/column. - Object desc = bank.getDescriptor(); - if (desc == null) { - throw new IllegalStateException("No descriptor and no schema entry for " + fullName); - } - - String typeName = tryGetTypeName(desc, col); - if (typeName != null) { - String s = typeName.trim(); - if (!s.isEmpty() && !s.equalsIgnoreCase("undefined")) { - return mapTypeName(s, fullName); - } - } - - Integer typeCode = tryGetTypeCode(desc, col); - if (typeCode != null) { - return mapTypeCode(typeCode, fullName); - } - - throw new IllegalStateException("Could not determine type for " + fullName); - } - - private Integer tryGetTypeCode(Object desc, String col) { - try { - Object v = desc.getClass() - .getMethod("getProperty", String.class, String.class) - .invoke(desc, "type", col); - if (v instanceof Number n) { - return n.intValue(); - } - } catch (ReflectiveOperationException ignored) { - } - - try { - Object v = desc.getClass() - .getMethod("getProperty", String.class) - .invoke(desc, col + ".type"); - if (v instanceof Number n) { - return n.intValue(); - } - } catch (ReflectiveOperationException ignored) { - } - - return null; - } - - private String tryGetTypeName(Object desc, String col) { - try { - Object v = desc.getClass() - .getMethod("getPropertyString", String.class, String.class) - .invoke(desc, "type", col); - if (v != null) { - return v.toString(); - } - } catch (ReflectiveOperationException ignored) { - } - - try { - Object v = desc.getClass() - .getMethod("getPropertyString", String.class) - .invoke(desc, col + ".type"); - if (v != null) { - return v.toString(); - } - } catch (ReflectiveOperationException ignored) { - } - - return null; - } - - private ColumnType mapTypeCode(int t, String fullName) { - return switch (t) { - case 1 -> ColumnType.INT; - case 2 -> ColumnType.FLOAT; - case 3 -> ColumnType.DOUBLE; - case 4 -> ColumnType.SHORT; - case 5 -> ColumnType.BYTE; - case 6 -> ColumnType.LONG; - case 8 -> ColumnType.BYTE; - default -> throw new IllegalStateException("Unknown type code " + t + " for " + fullName); - }; - } - - private ColumnType mapTypeName(String typeName, String fullName) { - String s = typeName.trim().toLowerCase(); - return switch (s) { - case "int", "int32", "i4" -> ColumnType.INT; - case "float", "float32", "f4" -> ColumnType.FLOAT; - case "double", "float64", "f8" -> ColumnType.DOUBLE; - case "short", "int16", "i2" -> ColumnType.SHORT; - case "byte", "int8", "i1" -> ColumnType.BYTE; - case "long", "int64", "i8" -> ColumnType.LONG; - default -> throw new IllegalStateException("Unknown type name '" + typeName + "' for " + fullName); - }; - } - - private void writeNpz(File output) throws IOException { - try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(output)))) { - zos.setLevel(Deflater.BEST_SPEED); - - for (BankStore bank : banks.values()) { - String bankBase = sanitize(bank.bankName); - - addEntry(zos, bankBase + "__rows_per_event.npy", - Npy.writeIntArray(bank.rowsPerEvent.toArray(), " indexSchemaFiles(File schemaDir) throws IOException { - File[] files = schemaDir.listFiles((dir, name) -> name.toLowerCase().endsWith(".json")); - if (files == null || files.length == 0) { - throw new IOException("No .json schema files found in " + schemaDir.getAbsolutePath()); - } - - Map bankToFile = new HashMap<>(); - for (File file : files) { - Object root = Json.parse(Files.readString(file.toPath(), StandardCharsets.UTF_8)); - if (!(root instanceof List banks)) { - continue; - } - - for (Object bankObj : banks) { - if (!(bankObj instanceof Map bankMap)) { - continue; - } - Object nameObj = bankMap.get("name"); - if (nameObj instanceof String bankName && !bankName.isBlank()) { - bankToFile.put(bankName, file); - } - } - } - return bankToFile; - } - - static Set chooseSchemaFiles(Map bankToFile, Set selectedBanks) { - Set files = new LinkedHashSet<>(); - - if (selectedBanks == null) { - files.addAll(bankToFile.values()); - return files; - } - - List missing = new ArrayList<>(); - for (String bank : selectedBanks) { - File file = bankToFile.get(bank); - if (file != null) { - files.add(file); - } else { - missing.add(bank); - } - } - - if (!missing.isEmpty()) { - System.err.println("Warning: no schema JSON file found for selected banks:"); - for (String b : missing) { - System.err.println(" " + b); - } - } - - return files; - } - - static Map loadSchemaTypes(Set schemaFiles) throws IOException { - Map types = new HashMap<>(); - - List sorted = new ArrayList<>(schemaFiles); - sorted.sort(Comparator.comparing(File::getName)); - - for (File file : sorted) { - Object root = Json.parse(Files.readString(file.toPath(), StandardCharsets.UTF_8)); - if (!(root instanceof List banks)) { - continue; - } - - for (Object bankObj : banks) { - if (!(bankObj instanceof Map bankMap)) { - continue; - } - - Object nameObj = bankMap.get("name"); - Object entriesObj = bankMap.get("entries"); - if (!(nameObj instanceof String bankName) || !(entriesObj instanceof List entries)) { - continue; - } - - for (Object entryObj : entries) { - if (!(entryObj instanceof Map entryMap)) { - continue; - } - - Object entryNameObj = entryMap.get("name"); - Object entryTypeObj = entryMap.get("type"); - if (!(entryNameObj instanceof String colName) || !(entryTypeObj instanceof String typeCode)) { - continue; - } - - String fullName = bankName + "/" + colName; - types.put(fullName, ColumnType.fromSchemaCode(typeCode, fullName)); - } - } - } - - return types; - } - } - - // ------------------------------------------------------------------------ - // Stores - // ------------------------------------------------------------------------ - - private static final class BankStore { - final String bankName; - final Map columns = new LinkedHashMap<>(); - final IntList rowsPerEvent = new IntList(); - final LongList offsets = new LongList(); - long totalRows = 0; - - BankStore(String bankName) { - this.bankName = bankName; - this.offsets.add(0L); - } - - void appendEventRows(int rows) { - rowsPerEvent.add(rows); - totalRows += rows; - offsets.add(totalRows); - } - } - - private static final class ColumnStore { - final String bankName; - final String columnName; - final ColumnType type; - final ByteList bytes; - final ShortList shorts; - final IntList ints; - final LongList longs; - final FloatList floats; - final DoubleList doubles; - - ColumnStore(String bankName, String columnName, ColumnType type) { - this.bankName = bankName; - this.columnName = columnName; - this.type = type; - this.bytes = type == ColumnType.BYTE ? new ByteList() : null; - this.shorts = type == ColumnType.SHORT ? new ShortList() : null; - this.ints = type == ColumnType.INT ? new IntList() : null; - this.longs = type == ColumnType.LONG ? new LongList() : null; - this.floats = type == ColumnType.FLOAT ? new FloatList() : null; - this.doubles = type == ColumnType.DOUBLE ? new DoubleList() : null; - } - - void append(DataBank bank, String col, int row) { - switch (type) { - case BYTE -> bytes.add(bank.getByte(col, row)); - case SHORT -> shorts.add(bank.getShort(col, row)); - case INT -> ints.add(bank.getInt(col, row)); - case LONG -> longs.add(bank.getLong(col, row)); - case FLOAT -> floats.add(bank.getFloat(col, row)); - case DOUBLE -> doubles.add(bank.getDouble(col, row)); - } - } - - byte[] toNpyBytes() throws IOException { - return switch (type) { - case BYTE -> Npy.writeByteArray(bytes.toArray(), type.npyDescr); - case SHORT -> Npy.writeShortArray(shorts.toArray(), type.npyDescr); - case INT -> Npy.writeIntArray(ints.toArray(), type.npyDescr); - case LONG -> Npy.writeLongArray(longs.toArray(), type.npyDescr); - case FLOAT -> Npy.writeFloatArray(floats.toArray(), type.npyDescr); - case DOUBLE -> Npy.writeDoubleArray(doubles.toArray(), type.npyDescr); - }; - } - } - - // ------------------------------------------------------------------------ - // Minimal JSON parser - // ------------------------------------------------------------------------ - - private static final class Json { - static Object parse(String text) { - return new Parser(text).parseValue(); - } - - private static final class Parser { - private final String s; - private int i = 0; - - Parser(String s) { - this.s = s; - } - - Object parseValue() { - skipWs(); - if (i >= s.length()) { - throw new IllegalStateException("Unexpected end of JSON"); - } - - char c = s.charAt(i); - return switch (c) { - case '{' -> parseObject(); - case '[' -> parseArray(); - case '"' -> parseString(); - case 't' -> parseTrue(); - case 'f' -> parseFalse(); - case 'n' -> parseNull(); - default -> parseNumber(); - }; - } - - private Map parseObject() { - expect('{'); - Map obj = new LinkedHashMap<>(); - skipWs(); - if (peek('}')) { - expect('}'); - return obj; - } - - while (true) { - skipWs(); - String key = parseString(); - skipWs(); - expect(':'); - Object value = parseValue(); - obj.put(key, value); - skipWs(); - if (peek('}')) { - expect('}'); - break; - } - expect(','); - } - return obj; - } - - private List parseArray() { - expect('['); - List arr = new ArrayList<>(); - skipWs(); - if (peek(']')) { - expect(']'); - return arr; - } - - while (true) { - arr.add(parseValue()); - skipWs(); - if (peek(']')) { - expect(']'); - break; - } - expect(','); - } - return arr; - } - - private String parseString() { - expect('"'); - StringBuilder sb = new StringBuilder(); - while (i < s.length()) { - char c = s.charAt(i++); - if (c == '"') { - return sb.toString(); - } - if (c == '\\') { - if (i >= s.length()) { - throw new IllegalStateException("Bad escape"); - } - char e = s.charAt(i++); - switch (e) { - case '"', '\\', '/' -> sb.append(e); - case 'b' -> sb.append('\b'); - case 'f' -> sb.append('\f'); - case 'n' -> sb.append('\n'); - case 'r' -> sb.append('\r'); - case 't' -> sb.append('\t'); - case 'u' -> { - if (i + 4 > s.length()) { - throw new IllegalStateException("Bad unicode escape"); - } - String hex = s.substring(i, i + 4); - sb.append((char) Integer.parseInt(hex, 16)); - i += 4; - } - default -> throw new IllegalStateException("Bad escape: \\" + e); - } - } else { - sb.append(c); - } - } - throw new IllegalStateException("Unterminated string"); - } - - private Object parseNumber() { - int start = i; - if (s.charAt(i) == '-') { - i++; - } - while (i < s.length() && Character.isDigit(s.charAt(i))) { - i++; - } - boolean isFloat = false; - if (i < s.length() && s.charAt(i) == '.') { - isFloat = true; - i++; - while (i < s.length() && Character.isDigit(s.charAt(i))) { - i++; - } - } - if (i < s.length() && (s.charAt(i) == 'e' || s.charAt(i) == 'E')) { - isFloat = true; - i++; - if (i < s.length() && (s.charAt(i) == '+' || s.charAt(i) == '-')) { - i++; - } - while (i < s.length() && Character.isDigit(s.charAt(i))) { - i++; - } - } - - String num = s.substring(start, i); - return isFloat ? Double.parseDouble(num) : Long.parseLong(num); - } - - private Boolean parseTrue() { - expect('t'); expect('r'); expect('u'); expect('e'); - return Boolean.TRUE; - } - - private Boolean parseFalse() { - expect('f'); expect('a'); expect('l'); expect('s'); expect('e'); - return Boolean.FALSE; - } - - private Object parseNull() { - expect('n'); expect('u'); expect('l'); expect('l'); - return null; - } - - private void skipWs() { - while (i < s.length()) { - char c = s.charAt(i); - if (c == ' ' || c == '\n' || c == '\r' || c == '\t') { - i++; - } else { - break; - } - } - } - - private boolean peek(char c) { - skipWs(); - return i < s.length() && s.charAt(i) == c; - } - - private void expect(char c) { - skipWs(); - if (i >= s.length() || s.charAt(i) != c) { - throw new IllegalStateException("Expected '" + c + "' at position " + i); - } - i++; - } - } - } - - // ------------------------------------------------------------------------ - // NPY writer - // ------------------------------------------------------------------------ - - private static final class Npy { - private static final byte[] MAGIC = {(byte) 0x93, 'N', 'U', 'M', 'P', 'Y'}; - - static byte[] writeByteArray(byte[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - out.write(values); - return out.toByteArray(); - } - - static byte[] writeShortArray(short[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 2).order(ByteOrder.LITTLE_ENDIAN); - for (short v : values) bb.putShort(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeIntArray(int[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (int v : values) bb.putInt(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeLongArray(long[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (long v : values) bb.putLong(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeFloatArray(float[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (float v : values) bb.putFloat(v); - out.write(bb.array()); - return out.toByteArray(); - } - - static byte[] writeDoubleArray(double[] values, String descr) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - writeHeader(out, descr, values.length); - ByteBuffer bb = ByteBuffer.allocate(values.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (double v : values) bb.putDouble(v); - out.write(bb.array()); - return out.toByteArray(); - } - - private static void writeHeader(ByteArrayOutputStream out, String descr, int length) throws IOException { - out.write(MAGIC); - out.write(1); - out.write(0); - - String dict = "{'descr': '" + descr + "', 'fortran_order': False, 'shape': (" + length + ",), }"; - int preamble = MAGIC.length + 2 + 2; - int padLen = 16 - ((preamble + dict.length() + 1) % 16); - if (padLen == 16) { - padLen = 0; - } - - String fullHeader = dict + " ".repeat(padLen) + "\n"; - byte[] fullHeaderBytes = fullHeader.getBytes(StandardCharsets.US_ASCII); - - ByteBuffer hlen = ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN); - hlen.putShort((short) fullHeaderBytes.length); - out.write(hlen.array()); - out.write(fullHeaderBytes); - } - } - - // ------------------------------------------------------------------------ - // Primitive dynamic arrays - // ------------------------------------------------------------------------ - - private static final class ByteList { - private byte[] data = new byte[1024]; - private int size = 0; - void add(byte v) { ensure(size + 1); data[size++] = v; } - byte[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class ShortList { - private short[] data = new short[1024]; - private int size = 0; - void add(short v) { ensure(size + 1); data[size++] = v; } - short[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class IntList { - private int[] data = new int[1024]; - private int size = 0; - void add(int v) { ensure(size + 1); data[size++] = v; } - int[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class LongList { - private long[] data = new long[1024]; - private int size = 0; - void add(long v) { ensure(size + 1); data[size++] = v; } - long[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class FloatList { - private float[] data = new float[1024]; - private int size = 0; - void add(float v) { ensure(size + 1); data[size++] = v; } - float[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } - - private static final class DoubleList { - private double[] data = new double[1024]; - private int size = 0; - void add(double v) { ensure(size + 1); data[size++] = v; } - double[] toArray() { return Arrays.copyOf(data, size); } - private void ensure(int n) { if (n > data.length) data = Arrays.copyOf(data, Math.max(n, data.length * 2)); } - } -} \ No newline at end of file diff --git a/list_npz_banks.py b/list_npz_banks.py deleted file mode 100644 index 930e930a9d..0000000000 --- a/list_npz_banks.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -""" -Show values for a few events from all banks in an NPZ produced by HipoToNpz. - -Usage: - python list_npz_banks.py file.npz - python list_npz_banks.py file.npz 3 -""" - -from __future__ import annotations - -import ast -import struct -import sys -from collections import defaultdict -from pathlib import Path -from zipfile import ZipFile - - -def read_npy_from_bytes(blob: bytes): - if len(blob) < 10 or blob[:6] != b"\x93NUMPY": - raise ValueError("Not a valid .npy payload") - - major = blob[6] - minor = blob[7] - - if major == 1: - header_len = struct.unpack(" 3: - print("Usage: python3 show_all_npz_events_no_numpy.py [N_EVENTS]") - sys.exit(2) - - npz_path = Path(sys.argv[1]) - n_events = int(sys.argv[2]) if len(sys.argv) == 3 else 3 - - if not npz_path.exists(): - print(f"File not found: {npz_path}") - sys.exit(1) - - arrays = load_npz(npz_path) - banks = collect_banks(arrays.keys()) - - for bank in banks: - rows_key = f"{bank}__rows_per_event" - offsets_key = f"{bank}__offsets" - - if rows_key not in arrays or offsets_key not in arrays: - continue - - rows_per_event = arrays[rows_key] - offsets = arrays[offsets_key] - - column_keys = sorted( - k for k in arrays.keys() - if k.startswith(bank + "__") and k not in (rows_key, offsets_key) - ) - - print("=" * 80) - print(f"BANK: {bank}") - print("COLUMNS:") - for key in column_keys: - print(f" {key[len(bank) + 2:]}") - print() - - nevt = min(n_events, len(rows_per_event)) - - for evt in range(nevt): - start = int(offsets[evt]) - stop = int(offsets[evt + 1]) - nrows = int(rows_per_event[evt]) - - print(f"Event {evt}: rows={nrows} slice=[{start}:{stop}]") - if nrows == 0: - print(" ") - print() - continue - - for key in column_keys: - col = key[len(bank) + 2:] - values = arrays[key][start:stop] - print(f" {col}: {list(values)}") - print() - - -if __name__ == "__main__": - main() From 0326e568ec2947b33acc712409fb3812b5fadc8b Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Mon, 27 Jul 2026 16:15:30 -0400 Subject: [PATCH 9/9] fix(build): dependency version --- reconstruction/cvt/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reconstruction/cvt/pom.xml b/reconstruction/cvt/pom.xml index 7a9db52551..168707823d 100644 --- a/reconstruction/cvt/pom.xml +++ b/reconstruction/cvt/pom.xml @@ -82,8 +82,8 @@ org.jlab.clas12.detector clas12detector-ai - 13.7.1-SNAPSHOT - + 14.1.2-SNAPSHOT + org.jlab.clas12.detector clas12detector-eb @@ -103,7 +103,7 @@ ai.djl api - +