FIREFLY-2006: Charts Grouping functionality

src/firefly/js/charts/TableStatsCntlr.js

@@ -27,6 +27,9 @@ export const TBLSTATS_DATA_KEY = 'tblstats';
 export const LOAD_TBL_STATS = `${TBLSTATS_DATA_KEY}/LOAD_TBL_STATS`;
 export const UPDATE_TBL_STATS = `${TBLSTATS_DATA_KEY}/UPDATE_TBL_STATS`;
 
+//Avoid duplicate column-stat requests while the first request for a table is still pending
+const pendingStatsTblIds = new Set();
+
 
 export default {actionCreators, reducers};
 
@@ -58,6 +61,8 @@ export function dispatchLoadTblStats(searchRequest, dispatcher= flux.process) {
 
     if (resultSetID !== resultSetIDNow
         || curNumericColCnt !== numericColCnt) {  // also reload stats if the number of numeric columns changes.
+        if (pendingStatsTblIds.has(tbl_id)) return;
+        pendingStatsTblIds.add(tbl_id);
         dispatcher({type: LOAD_TBL_STATS, payload: {searchRequest}});
     }
 }
@@ -196,7 +201,14 @@ function fetchTblStats(dispatch, activeTableServerRequest) {
                     colStats: undefined
                 }));
         }
-    );
+    ).finally(() => {
+        pendingStatsTblIds.delete(tbl_id);
+        const currentRequest = getTblById(tbl_id)?.request;
+        const currentResultSetID = get(getTblById(tbl_id), 'tableMeta.resultSetID');
+        if (currentRequest && currentResultSetID !== resultSetIDOnSubmit) {
+            dispatchLoadTblStats(currentRequest);
+        }
+    });
 }
 
 export function getColValStats(tblId) {

src/firefly/js/charts/dataTypes/FireflyGenericData.js

@@ -1,11 +1,12 @@
 /*
  * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt
  */
-import {get, isArray, isUndefined, uniqueId, isNil} from 'lodash';
+import {get, isArray, isEmpty, isUndefined, uniqueId, isNil} from 'lodash';
 import {getTblById, getColumns, getColumn, doFetchTable, stripColumnNameQuotes} from '../../tables/TableUtil.js';
 import {cloneRequest, makeSubQueryRequest, MAX_ROW} from '../../tables/TableRequestUtil.js';
+import {getFiltersAsSql} from '../../tables/FilterInfo.js';
 import {dispatchChartUpdate, dispatchError, getChartData, getTraceSymbol, hasUpperLimits, hasLowerLimits} from '../ChartsCntlr.js';
-import {formatColExpr, getDataChangesForMappings, updateHighlighted, updateSelection, isScatter2d, getMaxScatterRows, getMinScatterGLRows} from '../ChartUtil.js';
+import {formatColExpr, getDataChangesForMappings, getFilterResetAutorange, getOriginalRowIndexes, isCurrentTableSource, updateHighlighted, updateSelection, isScatter2d, getMaxScatterRows, getMinScatterGLRows} from '../ChartUtil.js';
 import {getTraceTSEntries as heatmapTSGetter} from './FireflyHeatmap.js';
 import {errorTypeFieldKey} from '../ui/options/Errors.jsx';
 
@@ -24,7 +25,7 @@ const numberOrArrayProps = ['marker.size'];
 export function getTraceTSEntries({traceTS, chartId, traceNum}) {
     const {mappings} = traceTS || {};
 
-    if (mappings) {
+    if (!isEmpty(mappings)) {
         const options = Object.assign({}, mappings);
 
         if (hasUpperLimits(chartId, traceNum) || hasLowerLimits(chartId, traceNum)) {
@@ -47,7 +48,7 @@ export function getTraceTSEntries({traceTS, chartId, traceNum}) {
 
 async function fetchData(chartId, traceNum, tablesource) {
 
-    const {tbl_id, mappings} = tablesource;
+    const {tbl_id, mappings, resetAxes} = tablesource;
     if (!mappings) {
         return;
     }
@@ -74,10 +75,12 @@ async function fetchData(chartId, traceNum, tablesource) {
     const sreq = createChartTblRequest(chartId, traceNum, tablesource);
 
     const tableModel = await doFetchTable(sreq).catch((reason) => {
-        dispatchError(chartId, traceNum, reason);
+        if (isCurrentTableSource(chartId, traceNum, tablesource)) dispatchError(chartId, traceNum, reason);
     });
 
-    if (tableModel.error) {
+    //ignore stale results from an older table source if this trace was reconnected while fetching
+    if (!isCurrentTableSource(chartId, traceNum, tablesource)) return;
+    if (tableModel?.error) {
         return dispatchError(chartId, traceNum, tableModel.error);
     }
 
@@ -87,6 +90,17 @@ async function fetchData(chartId, traceNum, tablesource) {
         // extra changes based on trace type
         addOtherChanges({changes, chartId, traceNum, tablesource, tableModel: originalTableModel});
 
+        if (resetAxes) {
+            const {layout} = getChartData(chartId) ?? {};
+            changes['layout.xaxis.autorange'] = getFilterResetAutorange(layout?.xaxis);
+            changes['layout.xaxis.range'] = undefined;
+            changes['layout.yaxis.autorange'] = getFilterResetAutorange(layout?.yaxis);
+            changes['layout.yaxis.range'] = undefined;
+        }
+
+        const rowIdx = getOriginalRowIndexes(tableModel);
+        if (rowIdx) changes[`data.${traceNum}.firefly.rowIdx`] = rowIdx;
+
         dispatchChartUpdate({chartId, changes});
         const {activeTrace} = getChartData(chartId);
         if (isUndefined(activeTrace) || activeTrace === traceNum) {
@@ -120,9 +134,13 @@ export function createChartTblRequest(chartId, traceNum, tablesource) {
         }).filter((c, i, a) => a.indexOf(c) === i).// remove duplicates
         join(', ')    // allows to use the same columns, ex. "w1" as "x", "w1" as "marker.color"
     }, true);
-    if (fireflyData?.[traceNum]?.filters) {
+    const groupFilter = fireflyData?.[traceNum]?.filters;
+    if (groupFilter) {
         const inclCols = (sreq.inclCols ? sreq.inclCols + ',' : '') + '"ROW_NUM" as "ORIG_IDX"';
-        sreq = makeSubQueryRequest(request, sreq.title, sreq.params,{filters: fireflyData?.[traceNum].filters, inclCols, pageSize: MAX_ROW});
+        const tableFilter = getFiltersAsSql(tbl_id);
+        //apply the table filter and this trace's group filter together
+        const filters = tableFilter ? `(${tableFilter}) AND (${groupFilter})` : groupFilter;
+        sreq = makeSubQueryRequest(request, sreq.title, sreq.params,{filters, inclCols, pageSize: MAX_ROW});
     }
     return sreq;
 }
@@ -212,7 +230,7 @@ export function addScatterChanges({changes, chartId, traceNum, tablesource, tabl
 
     const colors = get(changes, [`data.${traceNum}.marker.color`]);
     let cTipLabel = isArray(colors) ? get(mappings, 'marker.color') : '';
-    if (cTipLabel.length > 20) {
+    if (cTipLabel?.length > 20) {
         cTipLabel = 'c';
     }
 

src/firefly/js/charts/dataTypes/FireflyHeatmap.js

@@ -5,7 +5,7 @@ import {get, isArray} from 'lodash';
 import {getTblById, getColumn, doFetchTable, stripColumnNameQuotes} from '../../tables/TableUtil.js';
 import {makeTableFunctionRequest, MAX_ROW} from '../../tables/TableRequestUtil.js';
 import {dispatchChartUpdate, dispatchError, getChartData} from '../ChartsCntlr.js';
-import {isScatter2d, getMaxScatterRows, singleTraceUI, handleBigInt} from '../ChartUtil.js';
+import {isCurrentTableSource, isScatter2d, getMaxScatterRows, singleTraceUI, handleBigInt} from '../ChartUtil.js';
 import {serializeDecimateInfo, parseDecimateKey} from '../../tables/Decimate.js';
 import BrowserInfo from  '../../util/BrowserInfo.js';
 import {formatColExpr} from '../ChartUtil.js';
@@ -26,9 +26,13 @@ const DEFBINS = 100;
  * @param p.traceNum
  */
 export function getTraceTSEntries({traceTS, chartId, traceNum}) {
-    const {mappings} = traceTS;
+    const {mappings={}, options:traceOptions={}} = traceTS;
 
-    if (!mappings) return {};
+    //During partial chart updates, trace removal, pinning, filtering, or fullscreen transitions,
+    //the table source could temporarily contain empty mappings, which is why we need this fallback to traceOptions
+    const xColOrExpr = mappings.x || traceOptions.xColOrExpr;
+    const yColOrExpr = mappings.y || traceOptions.yColOrExpr;
+    if (!xColOrExpr || !yColOrExpr) return {};
 
     const {fireflyData, fireflyLayout} = getChartData(chartId) || {};
     // server call parameters
@@ -47,9 +51,7 @@ export function getTraceTSEntries({traceTS, chartId, traceNum}) {
     const ymax = get(fireflyLayout, 'yaxis.max');
 
     const options = {
-        xColOrExpr: get(mappings, 'x'),
-        yColOrExpr: get(mappings, 'y'),
-        maxbins, xyratio, xmin, xmax, ymin, ymax
+        xColOrExpr, yColOrExpr, maxbins, xyratio, xmin, xmax, ymin, ymax
     };
 
     return {options, fetchData};
@@ -75,7 +77,9 @@ function fetchData(chartId, traceNum, tablesource) {
     const numericCols = getColumns(tableModel, COL_TYPE.NUMBER).map((c) => c.name);
     const {request} = tableModel;
 
-    const {xColOrExpr, yColOrExpr, maxbins, xyratio, xmin, xmax, ymin, ymax} = options;
+    const {xColOrExpr, yColOrExpr, maxbins, xyratio, xmin, xmax, ymin, ymax} = options || {};
+    //A stale/reindexed source can briefly lack axis expressions, do not attempt to fetch in that case
+    if (!xColOrExpr || !yColOrExpr) return;
 
     const xColName = numericCols.includes(xColOrExpr) ? xColOrExpr : 'xColumnExpression';
     const asX = (xColName === xColOrExpr) ? '' : ` as "${xColName}"`;
@@ -94,6 +98,9 @@ function fetchData(chartId, traceNum, tablesource) {
         {decimate: serializeDecimateInfo(xColName, yColName, maxbins, xyratio, xmin, xmax, ymin, ymax, 0), pageSize: MAX_ROW});
 
     doFetchTable(req).then((tableModel) => {
+        if (!isCurrentTableSource(chartId, traceNum, tablesource)) {
+            return;
+        }
         if (tableModel.error) {
             dispatchError(chartId, traceNum, tableModel.error);
             return;
@@ -111,7 +118,7 @@ function fetchData(chartId, traceNum, tablesource) {
         }
     }).catch(
         (reason) => {
-            dispatchError(chartId, traceNum, reason);
+            if (isCurrentTableSource(chartId, traceNum, tablesource)) dispatchError(chartId, traceNum, reason);
         }
     );
 }
@@ -127,9 +134,9 @@ function getChanges({tableModel, tablesource, chartId, traceNum}) {
         return {};
     }
 
-    // default axes labels for the first trace (remove surrounding quotes, if any)
-    const xLabel = stripColumnNameQuotes(get(mappings, 'x'));
-    const yLabel = stripColumnNameQuotes(get(mappings, 'y'));
+    //prevent missing source mappings from breaking tooltip and axis-label formatting.
+    const xLabel = stripColumnNameQuotes(mappings?.x || 'x');
+    const yLabel = stripColumnNameQuotes(mappings?.y || 'y');
     const xTipLabel = xLabel.length > 20 ? xLabel.substring(0,18)+'...' : xLabel;
     const yTipLabel = yLabel.length > 20 ? yLabel.substring(0,18)+'...' : yLabel;
 
@@ -243,7 +250,11 @@ function getChanges({tableModel, tablesource, chartId, traceNum}) {
             if (singleTraceUI() || (data?.length===1)) {
                 changes[`data.${traceNum}.colorbar.title.text`] = 'pts';
             } else {
-                changes[`data.${traceNum}.colorbar.title.text`] = get(data, `${traceNum}.name`, 'pts');
+                const traceName = get(data, `${traceNum}.name`);
+                //automatically generated names (for example, "trace 1") are
+                //implementation details and are not useful colorbar titles
+                changes[`data.${traceNum}.colorbar.title.text`] =
+                    traceName && !/^trace\s+\d+$/i.test(traceName) ? traceName : 'pts';
             }
         }
 
@@ -304,6 +315,14 @@ export function adjustColorbars({data, fireflyData, layout}) {
     if (data) {
         const changes = {};
         const nbars = data.filter((d) => get(d, 'colorbar') && get(d, 'showscale', true)).length;
+        const fireflyColorbarCount = data.filter((d, i) =>
+            get(fireflyData, `${i}.fireflyColorbar`) && get(d, 'colorbar') && get(d, 'showscale', true)).length;
+        if (fireflyColorbarCount === 0 && nbars === 0 && layout?.legend?.orientation === 'h') {
+            //firefly colorbars use a horizontal legend while present. Clear
+            //that temporary override after the last colorbar is removed so
+            //plotly can restore its normal right-side legend placement
+            changes['layout.legend.orientation'] = undefined;
+        }
         const yside = get(layout, 'yaxis.side');
         const yOpposite = (yside === 'right');
         let cnt = 1;
@@ -337,4 +356,4 @@ export function addColorbarChanges(changes, yOpposite, traceNum, x=1.02) {
         changes[`data.${traceNum}.colorbar.xanchor`] = 'left';
         changes[`data.${traceNum}.colorbar.x`] = x>0 ? x : 1-x;
     }
-}
+}

src/firefly/js/charts/dataTypes/FireflyHistogram.js

@@ -6,7 +6,7 @@ import {logger} from '../../util/Logger.js';
 import {COL_TYPE, getColumn, getColumns, getTblById, doFetchTable, stripColumnNameQuotes} from '../../tables/TableUtil.js';
 import {cloneRequest, makeTableFunctionRequest, MAX_ROW} from '../../tables/TableRequestUtil.js';
 import {dispatchChartUpdate, dispatchError, getChartData} from '../ChartsCntlr.js';
-import {formatColExpr, handleBigInt} from '../ChartUtil.js';
+import {formatColExpr, handleBigInt, isCurrentTableSource} from '../ChartUtil.js';
 
 
 import {toMaxFixed, getDecimalPlaces} from '../../util/MathUtil.js';
@@ -22,15 +22,20 @@ import Color from '../../util/Color.js';
  * @param p.traceNum
  * @returns {{options: {}, fetchData: fetchData}}
  */
-export function getTraceTSEntries({chartId, traceNum}) {
+export function getTraceTSEntries({chartId, traceNum, traceTS={}}) {
 
     const options = {};
 
     // server call parameters
     const {fireflyData, layout} = getChartData(chartId) || {};
-    const histogramParams = get(fireflyData, `${traceNum}.options`);
+    //preserve histogram options when chart state updates are partial
+    const histogramParams = {
+        ...(traceTS.options || {}),
+        ...(fireflyData?.[traceNum]?.options || {})
+    };
+    if (!histogramParams.columnOrExpr) return {};
     options.columnExpression = histogramParams.columnOrExpr;
-    if (get(layout, 'xaxis.type') === 'log') {
+    if (layout?.xaxis?.type === 'log') {
         options.columnExpression = 'lg('+histogramParams.columnOrExpr+')';
     }
     if (histogramParams.fixedBinSizeSelection) { // fixed size bins
@@ -58,15 +63,28 @@ function fetchData(chartId, traceNum, tablesource) {
     const {tbl_id, options} = tablesource;
 
     const tableModel = getTblById(tbl_id);
+    if (!tableModel?.request || !options?.columnExpression) {
+        if (isCurrentTableSource(chartId, traceNum, tablesource)) {
+            dispatchError(chartId, traceNum, 'Histogram table source is incomplete');
+        }
+        return;
+    }
     const numericCols = getColumns(tableModel, COL_TYPE.NUMBER).map((c) => c.name);
 
     const {request} = tableModel;
     const valueColName = 'columnExpression';
+    const formattedColumnExpression = formatColExpr({
+        colOrExpr: options.columnExpression,
+        quoted: true,
+        colNames: numericCols
+    });
     const sreq = cloneRequest(request, {
         startIdx: 0,
         pageSize: MAX_ROW,
-        inclCols: `${formatColExpr({colOrExpr:options.columnExpression, quoted: true, colNames: numericCols})} as "${valueColName}"`,
-        sortInfo: `ASC,"${valueColName}"`
+        inclCols: `${formattedColumnExpression} as "${valueColName}"`,
+        //HistogramProcessor expects the selected value as "columnExpression",
+        //but the source query must sort by the original expression
+        sortInfo: `ASC,${formattedColumnExpression}`
     });
     const sreqTblId = uniqueId(request.tbl_id);
     sreq.META_INFO.tbl_id = sreqTblId;
@@ -80,6 +98,8 @@ function fetchData(chartId, traceNum, tablesource) {
     doFetchTable(req).then(
         (tableModel) => {
 
+            if (!isCurrentTableSource(chartId, traceNum, tablesource)) return;
+
             if (tableModel.error) {
                 dispatchError(chartId, traceNum, tableModel.error);
                 return;
@@ -147,7 +167,9 @@ function fetchData(chartId, traceNum, tablesource) {
         }
     ).catch(
         (reason) => {
-            dispatchError(chartId, traceNum, reason);
+            if (isCurrentTableSource(chartId, traceNum, tablesource)) {
+                dispatchError(chartId, traceNum, reason);
+            }
         }
     );
 }