From 7ecf40c09c802fb29183b57baed9e97708e90d52 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 19 Nov 2025 09:51:42 -0800 Subject: [PATCH 01/62] Initial removal --- .../labkey/api/exp/api/ExperimentService.java | 2 +- .../api/exp/api/SampleTypeDomainKind.java | 14 ++---- .../experiment/api/ExpMaterialTableImpl.java | 20 ++------ .../api/SampleTypeUpdateServiceDI.java | 50 ++++++++----------- 4 files changed, 31 insertions(+), 55 deletions(-) diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index 2ce76f42943..b1f3fd629cc 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -147,7 +147,7 @@ static void setInstance(ExperimentService impl) enum QueryOptions { - UseLsidForUpdate, + UseLsidForUpdate, // TODO: Use as marker for behaviors GetSampleRecomputeCol, SkipBulkRemapCache, DeferRequiredLineageValidation, diff --git a/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java b/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java index 4c58d2c7684..da47258672a 100644 --- a/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java +++ b/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java @@ -85,7 +85,7 @@ public class SampleTypeDomainKind extends AbstractDomainKind BASE_PROPERTIES; private static final Set INDEXES; @@ -105,7 +105,6 @@ public class SampleTypeDomainKind extends AbstractDomainKind MATERIAL_ALT_MERGE_KEYS; - public static final Set MATERIAL_ALT_UPDATE_KEYS; public static final List AMOUNT_RANGE_VALIDATORS = new ArrayList<>(); static { MATERIAL_ALT_MERGE_KEYS = Set.of(Column.MaterialSourceId.name(), Column.Name.name()); - MATERIAL_ALT_UPDATE_KEYS = Set.of(Column.LSID.name()); Lsid rangeValidatorLsid = DefaultPropertyValidator.createValidatorURI(PropertyValidatorType.Range); IPropertyValidator amountValidator = PropertyService.get().createValidator(rangeValidatorLsid.toString()); @@ -690,7 +688,7 @@ public TableInfo getLookupTableInfo() @Override protected ColumnInfo getPkColumn(TableInfo table) { - return t.getColumn("lsid"); + return t.getColumn("lsid"); // TODO: Seems to be pointing at the wrong table } }); } @@ -1632,7 +1630,6 @@ private SQLFragment getJoinSQL(Set selectedColumns) TableInfo provisioned = null == _ss ? null : _ss.getTinfo(); Set provisionedCols = new CaseInsensitiveHashSet(provisioned != null ? provisioned.getColumnNameSet() : Collections.emptySet()); provisionedCols.remove(Column.RowId.name()); - provisionedCols.remove(Column.LSID.name()); provisionedCols.remove(Column.Name.name()); boolean hasProvisionedColumns = containsProvisionedColumns(selectedColumns, provisionedCols); @@ -1662,7 +1659,6 @@ private SQLFragment getJoinSQL(Set selectedColumns) // don't select twice if ( Column.RowId.name().equalsIgnoreCase(propertyColumn.getColumnName()) || - Column.LSID.name().equalsIgnoreCase(propertyColumn.getColumnName()) || Column.Name.name().equalsIgnoreCase(propertyColumn.getColumnName()) ) { @@ -1839,19 +1835,9 @@ public CaseInsensitiveHashMap remapSchemaColumns() @Override public Set getAltMergeKeys(DataIteratorContext context) { - if (context.getInsertOption().updateOnly && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) - return getAltKeysForUpdate(); - return MATERIAL_ALT_MERGE_KEYS; } - @NotNull - @Override - public Set getAltKeysForUpdate() - { - return MATERIAL_ALT_UPDATE_KEYS; - } - @Override @NotNull public List> getAdditionalRequiredInsertColumns() @@ -1886,7 +1872,7 @@ public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorCon try { var persist = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _ss, getUserSchema().getContainer(), getUserSchema().getUser(), _ss.getImportAliasesIncludingAliquot(), sampleTypeObjectId) - .setFileLinkDirectory(SAMPLETYPE_FILE_DIRECTORY); + .setFileLinkDirectory(SAMPLE_TYPE_FILE_DIRECTORY_NAME); ExperimentServiceImpl experimentServiceImpl = ExperimentServiceImpl.get(); SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index d9e5a25995c..4557c02b31b 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -27,7 +27,6 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.assay.AssayFileWriter; import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.CaseInsensitiveMapWrapper; @@ -144,25 +143,13 @@ import static org.labkey.api.exp.api.ExpRunItem.PARENT_IMPORT_ALIAS_MAP_PROP; import static org.labkey.api.exp.api.ExperimentService.QueryOptions.SkipBulkRemapCache; import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_ROLLUP_FIELD_LABELS; +import static org.labkey.api.exp.api.SampleTypeDomainKind.SAMPLE_TYPE_FILE_DIRECTORY_NAME; import static org.labkey.api.exp.api.SampleTypeService.ConfigParameters.SkipAliquotRollup; import static org.labkey.api.exp.api.SampleTypeService.ConfigParameters.SkipMaxSampleCounterFunction; import static org.labkey.api.exp.api.SampleTypeService.MISSING_AMOUNT_ERROR_MESSAGE; import static org.labkey.api.exp.api.SampleTypeService.MISSING_UNITS_ERROR_MESSAGE; import static org.labkey.api.exp.api.SampleTypeService.UNPROVIDED_VALUE_ERROR_MESSAGE_PATTERN; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotedFromLSID; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.LSID; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Name; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RootMaterialRowId; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RowId; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.SampleState; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.*; import static org.labkey.api.exp.query.SamplesSchema.SCHEMA_SAMPLES; import static org.labkey.api.util.IntegerUtils.asLong; import static org.labkey.experiment.ExpDataIterators.incrementCounts; @@ -890,6 +877,12 @@ protected Map _update(User user, Container c, Map Date: Wed, 19 Nov 2025 16:31:15 -0800 Subject: [PATCH 02/62] More removal --- .../SampleUpdateAddColumnsDataIterator.java | 73 ++--- .../api/exp/query/ExpMaterialTable.java | 3 +- .../org/labkey/api/exp/query/ExpTable.java | 6 +- .../labkey/experiment/ExpDataIterators.java | 64 +++-- .../api/ExpDataClassDataTableImpl.java | 4 +- .../experiment/api/ExpMaterialTableImpl.java | 268 ++++++++---------- .../experiment/api/ExperimentServiceImpl.java | 5 +- .../api/SampleTypeUpdateServiceDI.java | 124 ++++---- 8 files changed, 272 insertions(+), 275 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java b/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java index d1e3197125f..3907cb8809d 100644 --- a/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java @@ -13,20 +13,22 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.util.StringUtilsLabKey; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotedFromLSID; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.MaterialSourceId; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Name; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RootMaterialRowId; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RowId; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.SampleState; import static org.labkey.api.util.IntegerUtils.asInteger; public class SampleUpdateAddColumnsDataIterator extends WrapperDataIterator { - public static final String ALIQUOTED_FROM_LSID_COLUMN_NAME = "AliquotedFromLSID"; - public static final String ROOT_ROW_ID_COLUMN_NAME = "RootMaterialRowId"; public static final String CURRENT_SAMPLE_STATUS_COLUMN_NAME = "_CurrentSampleState_"; - static final String KEY_COLUMN_NAME = "Name"; - static final String KEY_COLUMN_LSID = "LSID"; final CachingDataIterator _unwrapped; final TableInfo target; @@ -36,7 +38,6 @@ public class SampleUpdateAddColumnsDataIterator extends WrapperDataIterator final int _aliquotedFromColIndex; final int _rootMaterialRowIdColIndex; final int _currentSampleStateColIndex; - final boolean _useLsid; // prefetch of existing records int lastPrefetchRowNumber = -1; @@ -44,27 +45,23 @@ public class SampleUpdateAddColumnsDataIterator extends WrapperDataIterator final IntHashMap aliquotRoots = new IntHashMap<>(); final IntHashMap sampleState = new IntHashMap<>(); - public SampleUpdateAddColumnsDataIterator(DataIterator in, TableInfo target, long sampleTypeId, boolean useLsid) + public SampleUpdateAddColumnsDataIterator(DataIterator in, TableInfo target, long sampleTypeId, boolean useRowId) { super(in); - this._unwrapped = (CachingDataIterator)in; - this.target = target; - this._sampleTypeId = sampleTypeId; - this._useLsid = useLsid; var map = DataIteratorUtil.createColumnNameMap(in); - this._aliquotedFromColIndex = map.get(ALIQUOTED_FROM_LSID_COLUMN_NAME); - this._rootMaterialRowIdColIndex = map.get(ROOT_ROW_ID_COLUMN_NAME); + this._aliquotedFromColIndex = map.get(AliquotedFromLSID.name()); + this._rootMaterialRowIdColIndex = map.get(RootMaterialRowId.name()); this._currentSampleStateColIndex = map.get(CURRENT_SAMPLE_STATUS_COLUMN_NAME); - String keyCol = useLsid ? KEY_COLUMN_LSID : KEY_COLUMN_NAME; - Integer index = map.get(keyCol); - ColumnInfo col = target.getColumn(keyCol); + String keyColumnName = useRowId ? RowId.name() : Name.name(); + Integer index = map.get(keyColumnName); + ColumnInfo col = target.getColumn(keyColumnName); if (null == index || null == col) - throw new IllegalArgumentException("Key column not found: " + keyCol); + throw new IllegalArgumentException("Key column not found: " + keyColumnName); pkSupplier = in.getSupplier(index); pkColumn = col; } @@ -119,20 +116,28 @@ protected void prefetchExisting() throws BatchValidationException sampleState.clear(); int rowsToFetch = 50; - Map rowKeyMap = new LinkedHashMap<>(); - Map keyRowMap = new LinkedHashMap<>(); + String keyFieldName = pkColumn.getName(); + boolean numericKey = pkColumn.isNumericType(); + Map rowKeyMap = new LinkedHashMap<>(); + Map keyRowMap = new LinkedHashMap<>(); do { lastPrefetchRowNumber = asInteger(_delegate.get(0)); Object keyObj = pkSupplier.get(); - String key = null; - if (keyObj instanceof String) - key = StringUtilsLabKey.unquoteString((String) keyObj); + Object key = null; + if (keyObj instanceof String s) + key = numericKey ? Long.valueOf(s) : StringUtilsLabKey.unquoteString(s); else if (keyObj instanceof Number) - key = keyObj.toString(); - if (StringUtils.isEmpty(key)) - throw new IllegalArgumentException(KEY_COLUMN_NAME + " value not provided on row " + lastPrefetchRowNumber); + key = numericKey ? keyObj : keyObj.toString(); + + if (numericKey) + { + if (null == key) + throw new IllegalArgumentException(keyFieldName + " value not provided on row " + lastPrefetchRowNumber); + } + else if (StringUtils.isEmpty((String) key)) + throw new IllegalArgumentException(keyFieldName + " value not provided on row " + lastPrefetchRowNumber); rowKeyMap.put(lastPrefetchRowNumber, key); keyRowMap.put(key, lastPrefetchRowNumber); @@ -142,20 +147,19 @@ else if (keyObj instanceof Number) } while (--rowsToFetch > 0 && _delegate.next()); - String keyCol = _useLsid ? KEY_COLUMN_LSID : KEY_COLUMN_NAME; - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("MaterialSourceId"), _sampleTypeId); - FieldKey keyField = FieldKey.fromParts(keyCol); - filter.addCondition(keyField, rowKeyMap.values(), CompareType.IN); + SimpleFilter filter = new SimpleFilter(MaterialSourceId.fieldKey(), _sampleTypeId); + filter.addCondition(pkColumn.getFieldKey(), rowKeyMap.values(), CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), target.getUserSchema().getContainer()); - Map[] results = new TableSelector(ExperimentService.get().getTinfoMaterial(), Sets.newCaseInsensitiveHashSet(keyCol, "aliquotedfromlsid", "rootMaterialRowId", "sampleState"), filter, null).getMapArray(); + Set columns = Sets.newCaseInsensitiveHashSet(keyFieldName, AliquotedFromLSID.name(), RootMaterialRowId.name(), SampleState.name()); + Map[] results = new TableSelector(ExperimentService.get().getTinfoMaterial(), columns, filter, null).getMapArray(); for (Map result : results) { - String key = (String) result.get(keyCol); - Object aliquotedFromLSIDObj = result.get("aliquotedFromLSID"); - Object rootMaterialRowIdObj = result.get("rootMaterialRowId"); - Object sampleStateObj = result.get("sampleState"); + Object key = result.get(keyFieldName); + Object aliquotedFromLSIDObj = result.get(AliquotedFromLSID.name()); + Object rootMaterialRowIdObj = result.get(RootMaterialRowId.name()); + Object sampleStateObj = result.get(SampleState.name()); Integer rowInd = keyRowMap.get(key); if (aliquotedFromLSIDObj != null) aliquotParents.put(rowInd, (String) aliquotedFromLSIDObj); @@ -180,5 +184,4 @@ public boolean next() throws BatchValidationException prefetchExisting(); return ret; } - } diff --git a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java index 0956af9d020..280a6ce14aa 100644 --- a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java +++ b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java @@ -72,7 +72,8 @@ enum Column private boolean _hasUnit = false; private final String _label; - Column() { + Column() + { _label = ColumnInfo.labelFromName(name()); } diff --git a/api/src/org/labkey/api/exp/query/ExpTable.java b/api/src/org/labkey/api/exp/query/ExpTable.java index 9d81d735dbe..332eb882400 100644 --- a/api/src/org/labkey/api/exp/query/ExpTable.java +++ b/api/src/org/labkey/api/exp/query/ExpTable.java @@ -33,6 +33,7 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.security.permissions.Permission; +import java.util.Collections; import java.util.Set; public interface ExpTable extends ContainerFilterable, TableInfo @@ -125,16 +126,15 @@ default void setFilterPatterns(String columnName, String... patterns) // by default we do nothing } - /** returns a column that wraps objectid, this is only required to support the expObject() table method */ default ColumnInfo getExpObjectColumn() { return null; } - @Nullable default Set getAltMergeKeys(DataIteratorContext context) + @NotNull default Set getAltMergeKeys(DataIteratorContext context) { - return null; + return Collections.emptySet(); } class ExpObjectDataColumn extends DataColumn diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index f470308fb57..a6d3ac2f21f 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -105,7 +105,6 @@ import org.labkey.api.query.ValidationException; import org.labkey.api.reader.DataLoader; import org.labkey.api.reader.TabLoader; -import org.labkey.api.search.SearchService; import org.labkey.api.security.User; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.UpdatePermission; @@ -168,15 +167,15 @@ import static org.labkey.api.exp.api.ExpRunItem.INPUTS_PREFIX_LC; import static org.labkey.api.exp.api.ExperimentService.ALIASCOLUMNALIAS; import static org.labkey.api.exp.api.ExperimentService.QueryOptions.SkipBulkRemapCache; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; import static org.labkey.api.util.IntegerUtils.asLong; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Alias; import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotCount; import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotVolume; import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotedFromLSID; import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotCount; import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotVolume; import static org.labkey.api.exp.query.ExpMaterialTable.Column.Folder; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Flag; import static org.labkey.api.exp.query.ExpMaterialTable.Column.MaterialSourceId; import static org.labkey.api.exp.query.ExpMaterialTable.Column.Name; import static org.labkey.api.exp.query.ExpMaterialTable.Column.RootMaterialRowId; @@ -312,9 +311,9 @@ protected String validate(ColumnValidator v, int rowNum, Object value, DataItera Object aliquotedFromObj = data.get(_aliquotedFromColIdx); if (aliquotedFromObj != null) { - if (aliquotedFromObj instanceof String) + if (aliquotedFromObj instanceof String s) { - aliquotedFromValue = (String) aliquotedFromObj; + aliquotedFromValue = s; } else if (aliquotedFromObj instanceof Number) { @@ -1321,9 +1320,9 @@ else if (_nameCol != null) if (o != null) { - if (o instanceof String) + if (o instanceof String s) { - aliquotParentName = StringUtilsLabKey.unquoteString((String) o); + aliquotParentName = StringUtilsLabKey.unquoteString(s); } else if (o instanceof Number) { @@ -1345,7 +1344,7 @@ else if (o instanceof Number) for (Integer parentCol : _requiredParentCols.keySet()) { Object parentVal = get(parentCol); - if (parentVal == null || (parentVal instanceof String && ((String) parentVal).isEmpty())) + if (parentVal == null || (parentVal instanceof String s && s.isEmpty())) getErrors().addRowError(new ValidationException("Missing value for required property: " + _requiredParentCols.get(parentCol))); } } @@ -2204,7 +2203,7 @@ public static class PersistDataIteratorBuilder implements DataIteratorBuilder final Map _importAliases; // expTable is the shared experiment table e.g. exp.Data or exp.Materials - public PersistDataIteratorBuilder(@NotNull DataIteratorBuilder in, TableInfo expTable, TableInfo propsTable, ExpObject typeObject, Container container, User user, Map importAliases, @Nullable Long ownerObjectId) + public PersistDataIteratorBuilder(@NotNull DataIteratorBuilder in, TableInfo expTable, TableInfo propsTable, ExpObject typeObject, Container container, User user, Map importAliases) { _in = in; _expTable = expTable; @@ -2212,7 +2211,7 @@ public PersistDataIteratorBuilder(@NotNull DataIteratorBuilder in, TableInfo exp _dataTypeObject = typeObject; _container = container; _user = user; - _importAliases = importAliases != null ? new CaseInsensitiveHashMap<>(importAliases) : new CaseInsensitiveHashMap<>(); + _importAliases = importAliases != null ? new CaseInsensitiveHashMap<>(importAliases) : Collections.emptyMap(); } public PersistDataIteratorBuilder setIndexFunction(Function indexFunction) @@ -2256,13 +2255,12 @@ public DataIterator getDataIterator(DataIteratorContext context) boolean isSample = _expTable instanceof ExpMaterialTableImpl; SimpleTranslator step1 = new SimpleTranslator(input, context); - step1.selectAll(Sets.newCaseInsensitiveHashSet("alias"), _importAliases); - if (colNameMap.containsKey("alias")) - step1.addColumn(ExperimentService.ALIASCOLUMNALIAS, colNameMap.get("alias")); // see AliasDataIteratorBuilder + step1.selectAll(Sets.newCaseInsensitiveHashSet(Alias.name()), _importAliases); + if (colNameMap.containsKey(Alias.name())) + step1.addColumn(ExperimentService.ALIASCOLUMNALIAS, colNameMap.get(Alias.name())); // see AliasDataIteratorBuilder CaseInsensitiveHashSet dontUpdate = new CaseInsensitiveHashSet(); dontUpdate.addAll(NOT_FOR_UPDATE); - dontUpdate.add("rowid"); // rowid is added / not dropped for dataclass for QueryUpdateAuditEvent.rowpk audit purpose if (context.getInsertOption().updateOnly) { dontUpdate.add("objectid"); @@ -2271,46 +2269,50 @@ public DataIterator getDataIterator(DataIteratorContext context) } boolean isMergeOrUpdate = context.getInsertOption().allowUpdate; - boolean isUpdateUsingLsid = context.getInsertOption().updateOnly && colNameMap.containsKey("lsid") && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); CaseInsensitiveHashSet keyColumns = new CaseInsensitiveHashSet(); CaseInsensitiveHashSet propertyKeyColumns = new CaseInsensitiveHashSet(); if (!isMergeOrUpdate) keyColumns.add(ExpDataTable.Column.LSID.toString()); - NameExpressionOptionService svc = NameExpressionOptionService.get(); - boolean canUpdateNames = svc.getAllowUserSpecificNamesValue(_container); + boolean canUpdateNames = NameExpressionOptionService.get().getAllowUserSpecificNamesValue(_container); if (isSample) { + ExpMaterialTableImpl expMaterialTable = (ExpMaterialTableImpl) _expTable; + if (isMergeOrUpdate) { - if (isUpdateUsingLsid) + boolean isUpdateUsingRowId = context.getInsertOption().updateOnly && colNameMap.containsKey(RowId.name()); + if (isUpdateUsingRowId) { - keyColumns.add(ExpDataTable.Column.LSID.toString()); + keyColumns.add(RowId.name()); if (!canUpdateNames) - dontUpdate.add("name"); + dontUpdate.add(Name.name()); } else { - keyColumns.addAll(((ExpMaterialTableImpl) _expTable).getAltMergeKeys(context)); - propertyKeyColumns.add("name"); + keyColumns.addAll(expMaterialTable.getAltMergeKeys(context)); + propertyKeyColumns.add(Name.name()); } } - dontUpdate.addAll(((ExpMaterialTableImpl) _expTable).getUniqueIdFields()); - dontUpdate.add(RootMaterialRowId.toString()); - dontUpdate.add(AliquotedFromLSID.toString()); - dontUpdate.add(ExpMaterialTable.Column.AliquotCount.name()); - dontUpdate.add(ExpMaterialTable.Column.AliquotVolume.name()); - dontUpdate.add(ExpMaterialTable.Column.AvailableAliquotCount.name()); - dontUpdate.add(ExpMaterialTable.Column.AvailableAliquotVolume.name()); + dontUpdate.addAll(expMaterialTable.getUniqueIdFields()); + dontUpdate.addAll( + RootMaterialRowId.name(), + AliquotedFromLSID.name(), + AliquotCount.name(), + AliquotVolume.name(), + AvailableAliquotCount.name(), + AvailableAliquotVolume.name() + ); } else if (isMergeOrUpdate) { + boolean isUpdateUsingLsid = context.getInsertOption().updateOnly && colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); if (isUpdateUsingLsid) { - keyColumns.add(ExpDataTable.Column.LSID.toString()); + keyColumns.add(ExpDataTable.Column.LSID.name()); if (!canUpdateNames) dontUpdate.add("name"); } @@ -2360,7 +2362,7 @@ else if (isMergeOrUpdate) .setFailOnEmptyUpdate(false)); DataIteratorBuilder step5 = step4; - if (colNameMap.containsKey("flag") || colNameMap.containsKey("comment")) + if (colNameMap.containsKey(Flag.name()) || colNameMap.containsKey("comment")) { step5 = LoggingDataIterator.wrap(new ExpDataIterators.FlagDataIteratorBuilder(step4, _user, isSample, _dataTypeObject, _container)); } diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index 858833a24b9..8829eb21de5 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -881,7 +881,7 @@ public CaseInsensitiveHashMap remapSchemaColumns() } @Override - public Set getAltMergeKeys(DataIteratorContext context) + public @NotNull Set getAltMergeKeys(DataIteratorContext context) { if (context.getInsertOption().updateOnly && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) return getAltKeysForUpdate(); @@ -918,7 +918,7 @@ public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorCon TableInfo propertiesTable = _dataClassDataTableSupplier.get(); try { - PersistDataIteratorBuilder step0 = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _dataClass, getUserSchema().getContainer(), getUserSchema().getUser(), _dataClass.getImportAliases(), null); + PersistDataIteratorBuilder step0 = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _dataClass, getUserSchema().getContainer(), getUserSchema().getUser(), _dataClass.getImportAliases()); ExperimentServiceImpl experimentServiceImpl = ExperimentServiceImpl.get(); final var scope = propertiesTable.getSchema().getScope(); SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified); diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 6fda8482456..865a43dda8f 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -99,6 +99,7 @@ import org.labkey.api.query.UserSchema; import org.labkey.api.query.column.BuiltInColumnTypes; import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; import org.labkey.api.security.UserPrincipal; import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.InsertPermission; @@ -140,7 +141,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static java.util.Objects.requireNonNull; import static org.labkey.api.audit.AuditHandler.PROVIDED_DATA_PREFIX; import static org.labkey.api.data.ColumnRenderPropertiesImpl.NON_NEGATIVE_NUMBER_CONCEPT_URI; import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_COUNT_LABEL; @@ -148,16 +148,7 @@ import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_COUNT_LABEL; import static org.labkey.api.exp.api.SampleTypeDomainKind.AVAILABLE_ALIQUOT_VOLUME_LABEL; import static org.labkey.api.exp.api.SampleTypeDomainKind.SAMPLE_TYPE_FILE_DIRECTORY_NAME; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotUnit; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAliquotUnit; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAvailableAliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.*; import static org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult; import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.schema; @@ -170,7 +161,7 @@ public class ExpMaterialTableImpl extends ExpRunItemTableImpl MATERIAL_ALT_MERGE_KEYS; public static final List AMOUNT_RANGE_VALIDATORS = new ArrayList<>(); static { - MATERIAL_ALT_MERGE_KEYS = Set.of(Column.MaterialSourceId.name(), Column.Name.name()); + MATERIAL_ALT_MERGE_KEYS = Set.of(MaterialSourceId.name(), Name.name()); Lsid rangeValidatorLsid = DefaultPropertyValidator.createValidatorURI(PropertyValidatorType.Range); IPropertyValidator amountValidator = PropertyService.get().createValidator(rangeValidatorLsid.toString()); @@ -210,11 +201,11 @@ protected ColumnInfo resolveColumn(String name) if (result == null) { if ("CpasType".equalsIgnoreCase(name)) - result = createColumn(Column.SampleSet.name(), Column.SampleSet); - else if (Column.Property.name().equalsIgnoreCase(name)) - result = createPropertyColumn(Column.Property.name()); - else if (Column.QueryableInputs.name().equalsIgnoreCase(name)) - result = createColumn(Column.QueryableInputs.name(), Column.QueryableInputs); + result = createColumn(SampleSet.name(), SampleSet); + else if (Property.name().equalsIgnoreCase(name)) + result = createPropertyColumn(Property.name()); + else if (QueryableInputs.name().equalsIgnoreCase(name)) + result = createColumn(QueryableInputs.name(), QueryableInputs); } return result; } @@ -250,11 +241,11 @@ public MutableColumnInfo createColumn(String alias, Column column) } case LSID -> { - return wrapColumn(alias, _rootTable.getColumn(Column.LSID.name())); + return wrapColumn(alias, _rootTable.getColumn(LSID.name())); } case MaterialSourceId -> { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.MaterialSourceId.name())); + var columnInfo = wrapColumn(alias, _rootTable.getColumn(MaterialSourceId.name())); columnInfo.setFk(new LookupForeignKey(getLookupContainerFilter(), null, null, null, null, "RowId", "Name") { @Override @@ -278,8 +269,8 @@ public StringExpression getURL(ColumnInfo parent) } case RootMaterialRowId -> { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.RootMaterialRowId.name())); - columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.RowId.name())); + var columnInfo = wrapColumn(alias, _rootTable.getColumn(RootMaterialRowId.name())); + columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), RowId.name())); columnInfo.setLabel("Root Material"); columnInfo.setUserEditable(false); @@ -293,17 +284,17 @@ public StringExpression getURL(ColumnInfo parent) } case AliquotedFromLSID -> { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.AliquotedFromLSID.name())); + var columnInfo = wrapColumn(alias, _rootTable.getColumn(AliquotedFromLSID.name())); columnInfo.setSqlTypeName("lsidtype"); - columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), Column.LSID.name())); + columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), LSID.name())); columnInfo.setLabel("Aliquoted From Parent"); return columnInfo; } case IsAliquot -> { - String rootMaterialRowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RootMaterialRowId.name(); - String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); - ExprColumn columnInfo = new ExprColumn(this, FieldKey.fromParts(Column.IsAliquot.name()), new SQLFragment( + String rootMaterialRowIdField = ExprColumn.STR_TABLE_ALIAS + "." + RootMaterialRowId.name(); + String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + RowId.name(); + ExprColumn columnInfo = new ExprColumn(this, IsAliquot.fieldKey(), new SQLFragment( "(CASE WHEN (" + rootMaterialRowIdField + " = " + rowIdField + ") THEN ").append(getSqlDialect().getBooleanFALSE()) .append(" WHEN ").append(rowIdField).append(" IS NOT NULL THEN ").append(getSqlDialect().getBooleanTRUE()) // Issue 52745 .append(" ELSE NULL END)"), JdbcType.BOOLEAN); @@ -327,7 +318,7 @@ public StringExpression getURL(ColumnInfo parent) } case RawAmount -> { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); + var columnInfo = wrapColumn(alias, _rootTable.getColumn(StoredAmount.name())); columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, null)); columnInfo.setDescription("The amount of this sample, in the base unit for the sample type's display unit (if defined), currently on hand."); columnInfo.setUserEditable(false); @@ -356,7 +347,7 @@ public StringExpression getURL(ColumnInfo parent) } else { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.StoredAmount.name())); + var columnInfo = wrapColumn(alias, _rootTable.getColumn(StoredAmount.name())); columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, null)); columnInfo.setLabel(label); columnInfo.setImportAliasesSet(importAliases); @@ -368,7 +359,7 @@ public StringExpression getURL(ColumnInfo parent) } case RawUnits -> { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Units.name())); columnInfo.setDescription("The units associated with the Stored Amount for this sample."); columnInfo.setUserEditable(false); columnInfo.setReadOnly(true); @@ -388,7 +379,7 @@ public StringExpression getURL(ColumnInfo parent) Unit typeUnit = getSampleTypeUnit(); if (typeUnit != null) { - SampleTypeUnitDisplayColumn columnInfo = new SampleTypeUnitDisplayColumn(this, Column.Units.name(), typeUnit); + SampleTypeUnitDisplayColumn columnInfo = new SampleTypeUnitDisplayColumn(this, Units.name(), typeUnit); columnInfo.setFk(fk); columnInfo.setDescription("The sample type display units associated with the Amount for this sample."); columnInfo.setShownInUpdateView(true); @@ -399,7 +390,7 @@ public StringExpression getURL(ColumnInfo parent) } else { - var columnInfo = wrapColumn(alias, _rootTable.getColumn(Column.Units.name())); + var columnInfo = wrapColumn(alias, _rootTable.getColumn(Units.name())); columnInfo.setFk(fk); columnInfo.setDescription("The units associated with the Stored Amount for this sample."); return columnInfo; @@ -407,7 +398,7 @@ public StringExpression getURL(ColumnInfo parent) } case Description -> { - return wrapColumn(alias, _rootTable.getColumn(Column.Description.name())); + return wrapColumn(alias, _rootTable.getColumn(Description.name())); } case SampleSet -> { @@ -458,7 +449,7 @@ public StringExpression getURL(ColumnInfo parent) } case SourceApplicationInput -> { - var col = createEdgeColumn(alias, Column.SourceProtocolApplication, ExpSchema.TableType.MaterialInputs); + var col = createEdgeColumn(alias, SourceProtocolApplication, ExpSchema.TableType.MaterialInputs); col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's SourceProtocolApplication"); col.setHidden(true); return col; @@ -481,7 +472,7 @@ public StringExpression getURL(ColumnInfo parent) } case RunApplicationOutput -> { - var col = createEdgeColumn(alias, Column.RunApplication, ExpSchema.TableType.MaterialInputs); + var col = createEdgeColumn(alias, RunApplication, ExpSchema.TableType.MaterialInputs); col.setDescription("Contains a reference to the MaterialInput row between this ExpMaterial and it's RunOutputApplication"); return col; } @@ -650,7 +641,7 @@ public StringExpression getURL(ColumnInfo parent) } else { - var ret = wrapColumn(alias, _rootTable.getColumn("AliquotUnit")); + var ret = wrapColumn(alias, _rootTable.getColumn(AliquotUnit.name())); ret.setShownInDetailsView(false); return ret; } @@ -746,15 +737,15 @@ public ExpSampleType getSampleType() protected void populateColumns() { var st = getSampleType(); - var rowIdCol = addColumn(Column.RowId); - addColumn(Column.MaterialSourceId); - addColumn(Column.SourceProtocolApplication); - addColumn(Column.SourceApplicationInput); - addColumn(Column.RunApplication); - addColumn(Column.RunApplicationOutput); - addColumn(Column.SourceProtocolLSID); - - var nameCol = addColumn(Column.Name); + var rowIdCol = addColumn(RowId); + addColumn(MaterialSourceId); + addColumn(SourceProtocolApplication); + addColumn(SourceApplicationInput); + addColumn(RunApplication); + addColumn(RunApplicationOutput); + addColumn(SourceProtocolLSID); + + var nameCol = addColumn(Name); if (st != null && st.hasNameAsIdCol()) { // Show the Name field but don't mark is as required when using name expressions @@ -778,10 +769,10 @@ protected void populateColumns() nameCol.setShownInInsertView(false); } - addColumn(Column.Alias); - addColumn(Column.Description); + addColumn(Alias); + addColumn(Description); - var typeColumnInfo = addColumn(Column.SampleSet); + var typeColumnInfo = addColumn(SampleSet); typeColumnInfo.setFk(new QueryForeignKey(_userSchema, getContainerFilter(), ExpSchema.SCHEMA_NAME, getContainer(), null, ExpSchema.TableType.SampleSets.name(), "lsid", null) { @Override @@ -810,14 +801,14 @@ protected ContainerFilter getLookupContainerFilter() typeColumnInfo.setUserEditable(false); typeColumnInfo.setShownInInsertView(false); - addColumn(Column.MaterialExpDate); - addContainerColumn(Column.Folder, null); - var runCol = addColumn(Column.Run); + addColumn(MaterialExpDate); + addContainerColumn(Folder, null); + var runCol = addColumn(Run); runCol.setFk(new ExpSchema(_userSchema.getUser(), getContainer()).getRunIdForeignKey(getContainerFilter())); runCol.setShownInInsertView(false); runCol.setShownInUpdateView(false); - var colLSID = addColumn(Column.LSID); + var colLSID = addColumn(LSID); colLSID.setHidden(true); colLSID.setReadOnly(true); colLSID.setUserEditable(false); @@ -825,7 +816,7 @@ protected ContainerFilter getLookupContainerFilter() colLSID.setShownInDetailsView(false); colLSID.setShownInUpdateView(false); - var rootRowId = addColumn(Column.RootMaterialRowId); + var rootRowId = addColumn(RootMaterialRowId); rootRowId.setHidden(true); rootRowId.setReadOnly(true); rootRowId.setUserEditable(false); @@ -833,7 +824,7 @@ protected ContainerFilter getLookupContainerFilter() rootRowId.setShownInDetailsView(false); rootRowId.setShownInUpdateView(false); - var aliquotParentLSID = addColumn(Column.AliquotedFromLSID); + var aliquotParentLSID = addColumn(AliquotedFromLSID); aliquotParentLSID.setHidden(true); aliquotParentLSID.setReadOnly(true); aliquotParentLSID.setUserEditable(false); @@ -841,26 +832,26 @@ protected ContainerFilter getLookupContainerFilter() aliquotParentLSID.setShownInDetailsView(false); aliquotParentLSID.setShownInUpdateView(false); - addColumn(Column.IsAliquot); - addColumn(Column.Created); - addColumn(Column.CreatedBy); - addColumn(Column.Modified); - addColumn(Column.ModifiedBy); + addColumn(IsAliquot); + addColumn(Created); + addColumn(CreatedBy); + addColumn(Modified); + addColumn(ModifiedBy); List defaultCols = new ArrayList<>(); - defaultCols.add(FieldKey.fromParts(Column.Name)); - defaultCols.add(FieldKey.fromParts(Column.MaterialExpDate)); + defaultCols.add(Name.fieldKey()); + defaultCols.add(MaterialExpDate.fieldKey()); boolean hasProductFolders = getContainer().hasProductFolders(); if (hasProductFolders) - defaultCols.add(FieldKey.fromParts(Column.Folder)); - defaultCols.add(FieldKey.fromParts(Column.Run)); + defaultCols.add(Folder.fieldKey()); + defaultCols.add(Run.fieldKey()); if (st == null) - defaultCols.add(FieldKey.fromParts(Column.SampleSet)); + defaultCols.add(SampleSet.fieldKey()); - addColumn(Column.Flag); + addColumn(Flag); - var statusColInfo = addColumn(Column.SampleState); + var statusColInfo = addColumn(SampleState); boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); statusColInfo.setShownInDetailsView(statusEnabled); statusColInfo.setShownInInsertView(statusEnabled); @@ -868,14 +859,14 @@ protected ContainerFilter getLookupContainerFilter() statusColInfo.setHidden(!statusEnabled); statusColInfo.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); if (statusEnabled) - defaultCols.add(FieldKey.fromParts(Column.SampleState)); + defaultCols.add(SampleState.fieldKey()); statusColInfo.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); // TODO is this a real Domain??? if (st != null && !"urn:lsid:labkey.com:SampleSource:Default".equals(st.getDomain().getTypeURI())) { - defaultCols.add(FieldKey.fromParts(Column.Flag)); + defaultCols.add(Flag.fieldKey()); addSampleTypeColumns(st, defaultCols); setName(_ss.getName()); @@ -888,19 +879,19 @@ protected ContainerFilter getLookupContainerFilter() List calculatedFieldKeys = DomainUtil.getCalculatedFieldsForDefaultView(this); defaultCols.addAll(calculatedFieldKeys); - addColumn(Column.AliquotCount); - addColumn(Column.AliquotVolume); + addColumn(AliquotCount); + addColumn(AliquotVolume); addColumn(AliquotUnit); - addColumn(Column.AvailableAliquotCount); - addColumn(Column.AvailableAliquotVolume); + addColumn(AvailableAliquotCount); + addColumn(AvailableAliquotVolume); - addColumn(Column.StoredAmount); - defaultCols.add(FieldKey.fromParts(Column.StoredAmount)); + addColumn(StoredAmount); + defaultCols.add(StoredAmount.fieldKey()); - addColumn(Column.Units); - defaultCols.add(FieldKey.fromParts(Column.Units)); + addColumn(Units); + defaultCols.add(Units.fieldKey()); - var rawAmountColumn = addColumn(Column.RawAmount); + var rawAmountColumn = addColumn(RawAmount); rawAmountColumn.setDisplayColumnFactory(new DisplayColumnFactory() { @Override @@ -912,7 +903,7 @@ public DisplayColumn createRenderer(ColumnInfo colInfo) public void addQueryFieldKeys(Set keys) { super.addQueryFieldKeys(keys); - keys.add(FieldKey.fromParts(Column.StoredAmount)); + keys.add(StoredAmount.fieldKey()); } }; @@ -923,7 +914,7 @@ public void addQueryFieldKeys(Set keys) rawAmountColumn.setShownInInsertView(false); rawAmountColumn.setShownInUpdateView(false); - var rawUnitsColumn = addColumn(Column.RawUnits); + var rawUnitsColumn = addColumn(RawUnits); rawUnitsColumn.setDisplayColumnFactory(new DisplayColumnFactory() { @Override @@ -935,8 +926,7 @@ public DisplayColumn createRenderer(ColumnInfo colInfo) public void addQueryFieldKeys(Set keys) { super.addQueryFieldKeys(keys); - keys.add(FieldKey.fromParts(Column.Units)); - + keys.add(Units.fieldKey()); } }; } @@ -1030,7 +1020,7 @@ public void addQueryFieldKeys(Set keys) if (plateUserSchema != null && plateUserSchema.getTable("Well") != null) { - String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + Column.RowId.name(); + String rowIdField = ExprColumn.STR_TABLE_ALIAS + "." + RowId.name(); SQLFragment existsSubquery = new SQLFragment() .append("SELECT 1 FROM ") .append(plateUserSchema.getTable("Well"), "well") @@ -1047,7 +1037,7 @@ public void addQueryFieldKeys(Set keys) { sql = new SQLFragment("(SELECT NULL)"); } - var col = new ExprColumn(this, Column.IsPlated.name(), sql, JdbcType.VARCHAR); + var col = new ExprColumn(this, IsPlated.name(), sql, JdbcType.VARCHAR); col.setDescription("Whether the sample that has been plated, if plating is supported."); col.setUserEditable(false); col.setReadOnly(true); @@ -1055,17 +1045,17 @@ public void addQueryFieldKeys(Set keys) col.setShownInInsertView(false); col.setShownInUpdateView(false); if (plateUserSchema != null) - col.setURL(DetailsURL.fromString("plate-isPlated.api?sampleId=${" + Column.RowId.name() + "}")); + col.setURL(DetailsURL.fromString("plate-isPlated.api?sampleId=${" + RowId.name() + "}")); addColumn(col); addVocabularyDomains(); - addColumn(Column.Properties); + addColumn(Properties); - var colInputs = addColumn(Column.Inputs); + var colInputs = addColumn(Inputs); addMethod("Inputs", new LineageMethod(colInputs, true), Set.of(colInputs.getFieldKey())); - var colOutputs = addColumn(Column.Outputs); + var colOutputs = addColumn(Outputs); addMethod("Outputs", new LineageMethod(colOutputs, false), Set.of(colOutputs.getFieldKey())); addExpObjectMethod(); @@ -1091,11 +1081,11 @@ public void addQueryFieldKeys(Set keys) setUpdateURL(LINK_DISABLER); } - setTitleColumn(Column.Name.toString()); + setTitleColumn(Name.toString()); setDefaultVisibleColumns(defaultCols); - MutableColumnInfo lineageLookup = ClosureQueryHelper.createAncestorLookupColumnInfo("Ancestors", this, _rootTable.getColumn("rowid"), _ss, true); + MutableColumnInfo lineageLookup = ClosureQueryHelper.createAncestorLookupColumnInfo("Ancestors", this, _rootTable.getColumn(RowId.name()), _ss, true); addColumn(lineageLookup); } @@ -1121,7 +1111,6 @@ public Domain getDomain(boolean forUpdate) return _ss == null ? null : _ss.getDomain(forUpdate); } - public static String appendNameExpressionDescription(String currentDescription, String nameExpression, String nameExpressionPreview) { if (nameExpression == null) @@ -1156,11 +1145,10 @@ private void addSampleTypeColumns(ExpSampleType st, List visibleColumn UserSchema schema = getUserSchema(); Domain domain = st.getDomain(); - ColumnInfo rowIdColumn = getColumn(Column.RowId); - ColumnInfo lsidColumn = getColumn(Column.LSID); - ColumnInfo nameColumn = getColumn(Column.Name); + ColumnInfo rowIdColumn = getColumn(RowId); + ColumnInfo nameColumn = getColumn(Name); - visibleColumns.remove(FieldKey.fromParts(Column.Run.name())); + visibleColumns.remove(Run.fieldKey()); // When not using name expressions, mark the ID columns as required. // NOTE: If not explicitly set, the first domain property will be chosen as the ID column. @@ -1180,7 +1168,6 @@ private void addSampleTypeColumns(ExpSampleType st, List visibleColumn if ( rowIdColumn.getFieldKey().equals(dbColumn.getFieldKey()) || - lsidColumn.getFieldKey().equals(dbColumn.getFieldKey()) || nameColumn.getFieldKey().equals(dbColumn.getFieldKey()) ) { @@ -1203,7 +1190,7 @@ private void addSampleTypeColumns(ExpSampleType st, List visibleColumn if (null != dp) { PropertyColumn.copyAttributes(schema.getUser(), propColumn, dp.getPropertyDescriptor(), schema.getContainer(), - SchemaKey.fromParts("samples"), st.getName(), FieldKey.fromParts("RowId"), null, getLookupContainerFilter()); + SchemaKey.fromParts("samples"), st.getName(), RowId.fieldKey(), null, getLookupContainerFilter()); if (idCols.contains(dp)) { @@ -1263,9 +1250,9 @@ private void addSampleTypeColumns(ExpSampleType st, List visibleColumn if (selectedColumns.contains(new FieldKey(null, StoredAmount))) selectedColumns.add(new FieldKey(null, Units)); if (selectedColumns.contains(new FieldKey(null, ExpMaterial.ALIQUOTED_FROM_INPUT))) - selectedColumns.add(new FieldKey(null, Column.AliquotedFromLSID.name())); - if (selectedColumns.contains(new FieldKey(null, Column.IsAliquot.name()))) - selectedColumns.add(new FieldKey(null, Column.RootMaterialRowId.name())); + selectedColumns.add(new FieldKey(null, AliquotedFromLSID.name())); + if (selectedColumns.contains(new FieldKey(null, IsAliquot.name()))) + selectedColumns.add(new FieldKey(null, RootMaterialRowId.name())); if (selectedColumns.contains(new FieldKey(null, AliquotVolume.name())) || selectedColumns.contains(new FieldKey(null, AvailableAliquotVolume.name()))) selectedColumns.add(new FieldKey(null, AliquotUnit.name())); selectedColumns.addAll(wrappedFieldKeys); @@ -1629,8 +1616,8 @@ private SQLFragment getJoinSQL(Set selectedColumns) { TableInfo provisioned = null == _ss ? null : _ss.getTinfo(); Set provisionedCols = new CaseInsensitiveHashSet(provisioned != null ? provisioned.getColumnNameSet() : Collections.emptySet()); - provisionedCols.remove(Column.RowId.name()); - provisionedCols.remove(Column.Name.name()); + provisionedCols.remove(RowId.name()); + provisionedCols.remove(Name.name()); boolean hasProvisionedColumns = containsProvisionedColumns(selectedColumns, provisionedCols); boolean hasSampleColumns = false; @@ -1658,8 +1645,8 @@ private SQLFragment getJoinSQL(Set selectedColumns) { // don't select twice if ( - Column.RowId.name().equalsIgnoreCase(propertyColumn.getColumnName()) || - Column.Name.name().equalsIgnoreCase(propertyColumn.getColumnName()) + RowId.name().equalsIgnoreCase(propertyColumn.getColumnName()) || + Name.name().equalsIgnoreCase(propertyColumn.getColumnName()) ) { continue; @@ -1833,7 +1820,7 @@ public CaseInsensitiveHashMap remapSchemaColumns() } @Override - public Set getAltMergeKeys(DataIteratorContext context) + public @NotNull Set getAltMergeKeys(DataIteratorContext context) { return MATERIAL_ALT_MERGE_KEYS; } @@ -1862,46 +1849,42 @@ public DataIteratorBuilder persistRows(DataIteratorBuilder data, DataIteratorCon // The specimens sample type doesn't have a properties table if (propertiesTable == null) - { return data; - } - long sampleTypeObjectId = requireNonNull(getOwnerObjectId()); - - // TODO: subclass PersistDataIteratorBuilder to index Materials! not DataClass! try { - var persist = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _ss, getUserSchema().getContainer(), getUserSchema().getUser(), _ss.getImportAliasesIncludingAliquot(), sampleTypeObjectId) - .setFileLinkDirectory(SAMPLE_TYPE_FILE_DIRECTORY_NAME); - ExperimentServiceImpl experimentServiceImpl = ExperimentServiceImpl.get(); - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified); - - persist.setIndexFunction(searchIndexDataKeys -> propertiesTable.getSchema().getScope().addCommitTask(() -> - { - List lsids = searchIndexDataKeys.lsids(); - List orderedRowIds = searchIndexDataKeys.orderedRowIds(); - - // Issue 51263: order by RowId to reduce deadlock - ListUtils.partition(orderedRowIds, 100).forEach(sublist -> - queue.addRunnable((q) -> - { - for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterials(sublist)) - expMaterial.index(q, this); - }) - ); - - ListUtils.partition(lsids, 100).forEach(sublist -> - queue.addRunnable((q) -> - { - for (ExpMaterialImpl expMaterial : experimentServiceImpl.getExpMaterialsByLsid(sublist)) - expMaterial.index(q, this); - }) - ); - }, DbScope.CommitTaskOption.POSTCOMMIT) - ); - - DataIteratorBuilder builder = LoggingDataIterator.wrap(persist); - return LoggingDataIterator.wrap(new AliasDataIteratorBuilder(builder, getUserSchema().getContainer(), getUserSchema().getUser(), ExperimentService.get().getTinfoMaterialAliasMap(), _ss, true)); + final Container container = getContainer(); + final User user = getUserSchema().getUser(); + ExperimentServiceImpl expService = ExperimentServiceImpl.get(); + SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified); + + DataIteratorBuilder dib = new ExpDataIterators.PersistDataIteratorBuilder(data, this, propertiesTable, _ss, container, user, _ss.getImportAliasesIncludingAliquot()) + .setFileLinkDirectory(SAMPLE_TYPE_FILE_DIRECTORY_NAME) + .setIndexFunction(searchIndexDataKeys -> propertiesTable.getSchema().getScope().addCommitTask(() -> { + List lsids = searchIndexDataKeys.lsids(); + List orderedRowIds = searchIndexDataKeys.orderedRowIds(); + + // Issue 51263: order by RowId to reduce deadlock + ListUtils.partition(orderedRowIds, 100).forEach(sublist -> + queue.addRunnable((q) -> + { + for (ExpMaterialImpl expMaterial : expService.getExpMaterials(sublist)) + expMaterial.index(q, this); + }) + ); + + ListUtils.partition(lsids, 100).forEach(sublist -> + queue.addRunnable((q) -> + { + for (ExpMaterialImpl expMaterial : expService.getExpMaterialsByLsid(sublist)) + expMaterial.index(q, this); + }) + ); + }, DbScope.CommitTaskOption.POSTCOMMIT) + ); + + dib = LoggingDataIterator.wrap(dib); + return LoggingDataIterator.wrap(new AliasDataIteratorBuilder(dib, container, user, expService.getTinfoMaterialAliasMap(), _ss, true)); } catch (IOException e) { @@ -1978,10 +1961,11 @@ public void overlayMetadata(String tableName, UserSchema schema, Collection> insertRows(User user, Container container, List * presence or absence of columns in the incoming data. If both columns are present, no exception is thrown. * * @param columns The set of columns in the input - * @param allowsUpdate Whether the type of import supports updates or not */ - public static void confirmAmountAndUnitsColumns(Collection columns, boolean allowsUpdate) + public static void confirmAmountAndUnitsColumns(Collection columns) { boolean hasUnits = columns.stream().anyMatch(column -> column.equalsIgnoreCase(Units.name())); boolean hasAmount = columns.stream().anyMatch(column -> StoredAmount.namesAndLabels().contains(column)); @@ -525,25 +524,32 @@ public static void confirmAmountAndUnitsColumns(Collection columns, bool } @Override - public List> updateRows(User user, Container container, List> rows, List> oldKeys, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + public List> updateRows( + User user, + Container container, + List> rows, + List> oldKeys, + BatchValidationException errors, + @Nullable Map configParameters, + Map extraScriptContext + ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException { assert _sampleType != null : "SampleType required for insert/update, but not required for read/delete"; - if (rows != null && !rows.isEmpty()) - confirmAmountAndUnitsColumns(rows.get(0).keySet(), true); - + // TODO: Perhaps we invert and if an lsid is included we do row-by-row? For backwards compatibility. + // This would required joining in exp.material to get rowId or name. boolean useDib = false; - if (rows != null && !rows.isEmpty() && oldKeys == null) - useDib = rows.get(0).containsKey("lsid"); - - useDib = useDib && hasUniformKeys(rows); + if (rows != null && !rows.isEmpty()) + { + confirmAmountAndUnitsColumns(rows.get(0).keySet()); + useDib = hasUniformKeys(rows); + } List> results; DbScope scope = getSchema().getDbSchema().getScope(); if (useDib) { Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; - finalConfigParameters.put(ExperimentService.QueryOptions.UseLsidForUpdate, true); recordDataIteratorUsed(configParameters); try @@ -618,12 +624,12 @@ protected Map coerceTypes(Map row, Map getSampleMetaFields() Domain domain = getDomain(); Set fields = domain.getProperties().stream() .filter(dp -> !LSID.name().equalsIgnoreCase(dp.getName()) - && !ExpMaterialTable.Column.Name.name().equalsIgnoreCase(dp.getName()) + && !Name.name().equalsIgnoreCase(dp.getName()) && (StringUtils.isEmpty(dp.getDerivationDataScope()) || ExpSchema.DerivationDataScopeType.ParentOnly.name().equalsIgnoreCase(dp.getDerivationDataScope()))) .map(ImportAliasable::getName) @@ -993,7 +999,7 @@ else if (!isAliquot && isAliquotField) validRowCopy.put(updateField, updateValue); } - if (ExpMaterialTable.Column.SampleState.name().equalsIgnoreCase(updateField)) + if (SampleState.name().equalsIgnoreCase(updateField)) hasStatusCol = true; } // had a locked status before and either not updating the status or updating to a new locked status @@ -1163,12 +1169,12 @@ public List> deleteRows(User user, Container container, List private @Nullable Integer getMaterialSourceId(Map row) { - return getMaterialIntegerValue(row, ExpMaterialTable.Column.MaterialSourceId.name()); + return getMaterialIntegerValue(row, MaterialSourceId.name()); } private @Nullable Long getMaterialRowId(Map row) { - return MapUtils.getLong(row, ExpMaterialTable.Column.RowId.name()); + return MapUtils.getLong(row, RowId.name()); } private Map getMaterialMap(Long rowId, String lsid, User user, Container container, boolean addInputs) @@ -1176,7 +1182,7 @@ private Map getMaterialMap(Long rowId, String lsid, User user, C { Filter filter; if (rowId != null) - filter = new SimpleFilter(ExpMaterialTable.Column.RowId.fieldKey(), rowId); + filter = new SimpleFilter(RowId.fieldKey(), rowId); else if (lsid != null) filter = new SimpleFilter(LSID.fieldKey(), lsid); else @@ -1212,8 +1218,8 @@ public boolean hasExistingRowsInOtherContainers(Container container, Map> getExistingRows(User user, Container container, Map> keys, boolean verifyNoCrossFolderData, boolean verifyExisting, @Nullable Set columns) - throws InvalidKeyException, QueryUpdateServiceException + public Map> getExistingRows( + User user, + Container container, + Map> keys, + boolean verifyNoCrossFolderData, + boolean verifyExisting, + @Nullable Set columns + ) throws InvalidKeyException, QueryUpdateServiceException { ExistingRowSelect existingRowSelect = getExistingRowSelect(columns); TableInfo queryTableInfo = existingRowSelect.tableInfo; @@ -1296,6 +1308,7 @@ public Map> getExistingRows(User user, Container co Map rowNumLsid = new IntHashMap<>(); Map rowIdRowNumMap = new LinkedHashMap<>(); + // TODO: What if we didn't support lsidRowMap? Map lsidRowNumMap = new CaseInsensitiveMapWrapper<>(new LinkedHashMap<>()); Map nameRowNumMap = new LinkedHashMap<>(); Integer sampleTypeId = null; @@ -1321,12 +1334,12 @@ else if (name != null && materialSourceId != null) nameRowNumMap.put(name, rowNum); } else - throw new QueryUpdateServiceException("Either RowId or LSID is required to get Sample Type Material."); + throw new QueryUpdateServiceException("Either RowId or Name is required to get Sample Type Material."); } if (!rowIdRowNumMap.isEmpty()) { - SimpleFilter filter = new SimpleFilter(ExpMaterialTable.Column.RowId.fieldKey(), rowIdRowNumMap.keySet(), CompareType.IN); + SimpleFilter filter = new SimpleFilter(RowId.fieldKey(), rowIdRowNumMap.keySet(), CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), container); Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); for (Map row : rows) @@ -1348,7 +1361,7 @@ else if (name != null && materialSourceId != null) useLsid = true; allKeys.addAll(lsidRowNumMap.keySet()); - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(LSID), lsidRowNumMap.keySet(), CompareType.IN); + SimpleFilter filter = new SimpleFilter(LSID.fieldKey(), lsidRowNumMap.keySet(), CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), container); Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); for (Map row : rows) @@ -1364,8 +1377,8 @@ else if (name != null && materialSourceId != null) if (!nameRowNumMap.isEmpty()) { allKeys.addAll(nameRowNumMap.keySet()); - SimpleFilter filter = new SimpleFilter(ExpMaterialTable.Column.MaterialSourceId.fieldKey(), sampleTypeId); - filter.addCondition(ExpMaterialTable.Column.Name.fieldKey(), nameRowNumMap.keySet(), CompareType.IN); + SimpleFilter filter = new SimpleFilter(MaterialSourceId.fieldKey(), sampleTypeId); + filter.addCondition(Name.fieldKey(), nameRowNumMap.keySet(), CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), container); Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); for (Map row : rows) @@ -1385,11 +1398,12 @@ else if (name != null && materialSourceId != null) // Issue 52922: cross folder merge without Product Folders enabled silently ignores the cross folder row update ContainerFilter allCf = new ContainerFilter.AllInProjectPlusShared(container, user); // use a relaxed CF to find existing data from cross containers - SimpleFilter existingDataFilter = new SimpleFilter(ExpMaterialTable.Column.MaterialSourceId.fieldKey(), sampleTypeId); - existingDataFilter.addCondition(FieldKey.fromParts("Container"), allCf.getIds(), CompareType.IN); + SimpleFilter existingDataFilter = new SimpleFilter(MaterialSourceId.fieldKey(), sampleTypeId); + existingDataFilter.addCondition(allCf.createFilterClause(ExperimentService.get().getSchema(), FieldKey.fromParts("Container"))); + existingDataFilter.addCondition(useLsid ? LSID.fieldKey() : Name.fieldKey(), allKeys, CompareType.IN); - existingDataFilter.addCondition(FieldKey.fromParts(useLsid ? "LSID" : "Name"), allKeys, CompareType.IN); - Map[] cfRows = new TableSelector(ExperimentService.get().getTinfoMaterial(), existingDataFilter, null).getMapArray(); + // TODO: Couldn't this question be asked in the query and return a max of one row where the container does not match? + Map[] cfRows = new TableSelector(ExperimentService.get().getTinfoMaterial(), Sets.newCaseInsensitiveHashSet("Container", Name.name()), existingDataFilter, null).getMapArray(); for (Map row : cfRows) { String dataContainer = (String) row.get("container"); @@ -1422,7 +1436,7 @@ else if (name != null && materialSourceId != null) Set lsids = new HashSet<>(); for (Map sampleRow : sampleRows.values()) - lsids.add((String) sampleRow.get("lsid")); + lsids.add(getMaterialLsid(sampleRow)); List seeds = ExperimentServiceImpl.get().getExpMaterialsByLsid(lsids); ExperimentServiceImpl.get().addRowsParentsFields(new HashSet<>(seeds), sampleRows, user, container); @@ -1564,6 +1578,7 @@ public PrepareDataIteratorBuilder(@NotNull ExpSampleTypeImpl sampleType, ExpMate public DataIterator getDataIterator(DataIteratorContext context) { DataIterator source = LoggingDataIterator.wrap(builder.getDataIterator(context)); + boolean isUpdate = context.getInsertOption() == InsertOption.UPDATE; // drop columns ColumnInfo containerColumn = this.materialTable.getColumn(this.materialTable.getContainerFieldKey()); @@ -1599,6 +1614,8 @@ public DataIterator getDataIterator(DataIteratorContext context) continue; if (isContainerField && context.isCrossFolderImport() && !context.getInsertOption().updateOnly) continue; + if (isUpdate && RowId.name().equalsIgnoreCase(name)) + continue; drop.add(name); } } @@ -1609,7 +1626,7 @@ public DataIterator getDataIterator(DataIteratorContext context) source = new DropColumnsDataIterator(source, drop); Map columnNameMap = DataIteratorUtil.createColumnNameMap(source); - if (context.getInsertOption() == InsertOption.UPDATE) + if (isUpdate) { SimpleTranslator addAliquotedFrom = new SimpleTranslator(source, context); @@ -1624,7 +1641,7 @@ public DataIterator getDataIterator(DataIteratorContext context) addAliquotedFrom.addNullColumn(PARENT_RECOMPUTE_NAME_COL, JdbcType.VARCHAR); addAliquotedFrom.selectAll(); - var addRequiredColsDI = new SampleUpdateAddColumnsDataIterator(new CachingDataIterator(addAliquotedFrom), materialTable, sampleType.getRowId(), columnNameMap.containsKey("lsid")); + var addRequiredColsDI = new SampleUpdateAddColumnsDataIterator(new CachingDataIterator(addAliquotedFrom), materialTable, sampleType.getRowId(), columnNameMap.containsKey(RowId.name())); SimpleTranslator c = new _SamplesCoerceDataIterator(addRequiredColsDI, context, sampleType, materialTable); context.setWithLookupRemapping(false); @@ -1672,7 +1689,7 @@ private static boolean isReservedHeader(String name) return true; if (ExperimentService.isInputOutputColumn(name)) return true; - for (ExpMaterialTable.Column column : ExpMaterialTable.Column.values()) + for (ExpMaterialTable.Column column : values()) { if (isExpMaterialColumn(column, name)) return true; @@ -1687,12 +1704,12 @@ private static boolean isExpMaterialColumn(ExpMaterialTable.Column column, Strin private static boolean isNameHeader(String name) { - return isExpMaterialColumn(ExpMaterialTable.Column.Name, name); + return isExpMaterialColumn(Name, name); } private static boolean isDescriptionHeader(String name) { - return isExpMaterialColumn(ExpMaterialTable.Column.Description, name); + return isExpMaterialColumn(Description, name); } private static boolean isSampleStateHeader(String name) @@ -1702,27 +1719,27 @@ private static boolean isSampleStateHeader(String name) private static boolean isCommentHeader(String name) { - return isExpMaterialColumn(ExpMaterialTable.Column.Flag, name) || "Comment".equalsIgnoreCase(name); + return isExpMaterialColumn(Flag, name) || "Comment".equalsIgnoreCase(name); } private static boolean isAliasHeader(String name) { - return isExpMaterialColumn(ExpMaterialTable.Column.Alias, name); + return isExpMaterialColumn(Alias, name); } private static boolean isMaterialExpDateHeader(String name) { - return isExpMaterialColumn(ExpMaterialTable.Column.MaterialExpDate, name); + return isExpMaterialColumn(MaterialExpDate, name); } private static boolean isStoredAmountHeader(String name) { - return isExpMaterialColumn(ExpMaterialTable.Column.StoredAmount, name) || "Amount".equalsIgnoreCase(name); + return isExpMaterialColumn(StoredAmount, name) || StoredAmount.label().equalsIgnoreCase(name); } public static boolean isUnitsHeader(String name) { - return isExpMaterialColumn(ExpMaterialTable.Column.Units, name); + return isExpMaterialColumn(Units, name); } private static boolean isAliquotRollupHeader(String name) @@ -1737,7 +1754,6 @@ private static boolean isAliquotRollupHeader(String name) static class _GenerateNamesDataIterator extends SimpleTranslator { final boolean _allowUserSpecifiedNames; // whether manual names specification is allowed or only name expression generation - final int _batchSize; final RemapCache _cache; final Container _container; final List>> _extraPropsFns; @@ -1786,20 +1802,12 @@ static class _GenerateNamesDataIterator extends SimpleTranslator boolean skipDuplicateCheck = context.getConfigParameterBoolean(SkipMaxSampleCounterFunction); _nameState = sampleType.getNameGenState(skipDuplicateCheck, true, _container, user); lsidBuilder = sampleType.generateSampleLSID(); - _batchSize = batchSize; - - boolean useLsidForUpdate = context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); - if (useLsidForUpdate) - selectAll(CaseInsensitiveHashSet.of(Name.name(), RootMaterialRowId.name())); - else - selectAll(CaseInsensitiveHashSet.of(Name.name(), LSID.name(), RootMaterialRowId.name())); + _lsidDbSeq = sampleType.getSampleLsidDbSeq(batchSize, sampleType.getContainer()); - _lsidDbSeq = sampleType.getSampleLsidDbSeq(_batchSize, sampleType.getContainer()); + selectAll(CaseInsensitiveHashSet.of(Name.name(), LSID.name(), RootMaterialRowId.name())); addColumn(new BaseColumnInfo("name", JdbcType.VARCHAR), (Supplier)() -> generatedName); - if (!useLsidForUpdate) - addColumn(new BaseColumnInfo("lsid", JdbcType.VARCHAR), (Supplier)() -> lsidBuilder.setObjectId(String.valueOf(_lsidDbSeq.next())).toString()); - // Ensure we have a cpasType column and it is of the right value + addColumn(new BaseColumnInfo("lsid", JdbcType.VARCHAR), (Supplier)() -> lsidBuilder.setObjectId(String.valueOf(_lsidDbSeq.next())).toString()); addColumn(new BaseColumnInfo("cpasType", JdbcType.VARCHAR), new SimpleTranslator.ConstantColumn(sampleType.getLSID())); addColumn(new BaseColumnInfo("materialSourceId", JdbcType.INTEGER), new SimpleTranslator.ConstantColumn(sampleType.getRowId())); } From 8bbd758b83d7d76c92c2fef72647ce70e33a5763 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 20 Nov 2025 15:17:43 -0800 Subject: [PATCH 03/62] Various updates --- .../labkey/experiment/api/ExpMaterialImpl.java | 15 +++++++-------- .../experiment/api/ExperimentServiceImpl.java | 7 ++++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java index 8a59ed07001..7302e552b18 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java @@ -47,7 +47,6 @@ import org.labkey.api.exp.api.SampleTypeService; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.query.ExpDataTable; import org.labkey.api.exp.query.ExpMaterialTable; import org.labkey.api.exp.query.ExpSchema; import org.labkey.api.exp.query.SamplesSchema; @@ -125,9 +124,9 @@ public ActionURL detailsURL(Container container, boolean checkForOverride) { ExpSampleType st = getSampleType(); if (st != null) - return new QueryRowReference(getContainer(), SamplesSchema.SCHEMA_SAMPLES, st.getName(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); + return new QueryRowReference(getContainer(), SamplesSchema.SCHEMA_SAMPLES, st.getName(), ExpMaterialTable.Column.RowId.fieldKey(), getRowId()); else - return new QueryRowReference(getContainer(), ExpSchema.SCHEMA_EXP, ExpSchema.TableType.Materials.name(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); + return new QueryRowReference(getContainer(), ExpSchema.SCHEMA_EXP, ExpSchema.TableType.Materials.name(), ExpMaterialTable.Column.RowId.fieldKey(), getRowId()); } @Override @@ -319,7 +318,7 @@ public void save(User user, ExpSampleTypeImpl st) TableInfo ti = st.getTinfo(); if (null != ti) { - new SqlExecutor(ti.getSchema()).execute("INSERT INTO " + ti + " (rowId, lsid, name) SELECT ?, ?, ? WHERE NOT EXISTS (SELECT lsid FROM " + ti + " WHERE lsid = ?)", getRowId(), getLSID(), getName(), getLSID()); + new SqlExecutor(ti.getSchema()).execute("INSERT INTO " + ti + " (rowId, name) SELECT ?, ? WHERE NOT EXISTS (SELECT rowId FROM " + ti + " WHERE rowId = ?)", getRowId(), getName(), getRowId()); SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(st, SampleTypeServiceImpl.SampleChangeType.insert); } } @@ -335,7 +334,7 @@ protected void save(User user, TableInfo table, boolean ensureObject) if (getRowId() == 0) { isInsert = true; - long longId = DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), ExperimentService.get().getTinfoMaterial().getDbSequenceName("RowId")).next(); + long longId = DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), ExperimentService.get().getTinfoMaterial().getDbSequenceName(ExpMaterialTable.Column.RowId.name())).next(); if (longId > Integer.MAX_VALUE) throw new OutOfRangeException(longId, 0, Integer.MAX_VALUE); setRowId((int) longId); @@ -507,7 +506,7 @@ public Map getProperties(ExpSampleTypeImpl st) var ti = null == st ? null : st.getTinfo(); if (null != ti) { - var selector = new TableSelector(ti, TableSelector.ALL_COLUMNS, new SimpleFilter(FieldKey.fromParts("lsid"), getLSID()), null); + var selector = new TableSelector(ti, new SimpleFilter(ExpMaterialTable.Column.RowId.fieldKey(), getRowId()), null); selector.forEach(rs -> { for (ColumnInfo c : ti.getColumns()) @@ -629,8 +628,8 @@ else if (values.containsKey(dp.getPropertyURI())) values.remove(key); } TableInfo tableInfo = st.getTinfo(); - ColumnInfo lsidCol = tableInfo.getColumn(ExpMaterialTable.Column.LSID.name()); - Table.update(user, st.getTinfo(), converted, lsidCol, getLSID(), null, Level.WARN); + ColumnInfo rowIdCol = tableInfo.getColumn(ExpMaterialTable.Column.RowId.fieldKey()); + Table.update(user, tableInfo, converted, rowIdCol, getRowId(), null, Level.WARN); } for (var entry : values.entrySet()) { diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 47e0760c74d..aa3392fb246 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -9921,10 +9921,11 @@ public int moveAuditEvents(Container targetContainer, List runLsids) continue; } query.append(unionAll); - query.append("SELECT LSID, ") - .append("CAST (").appendIdentifier(col.getSelectIdentifier()).append(" AS VARCHAR)") + query.append("SELECT M.LSID, ") + .append("CAST (ST.").appendIdentifier(col.getSelectIdentifier()).append(" AS VARCHAR)") .append(" AS ").append(UNIQUE_ID_COL_NAME); - query.append(" FROM expsampleset.").append(dialect.quoteIdentifier(provisioned.getName())); + query.append(" FROM ").append(provisioned, "ST"); + query.append(" INNER JOIN ").append(ExperimentService.get().getTinfoMaterial(), "M").append(" ON M.RowId = ST.RowId"); query.append(" WHERE ").appendIdentifier(col.getSelectIdentifier()).appendInClause(isIntegerField ? intIds : uniqueIds, dialect); unionAll = "\n UNION ALL\n"; } From 7d2b57bcfa5906f00f73bb1b1015711bf57cf6a1 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 21 Nov 2025 09:19:38 -0800 Subject: [PATCH 04/62] getObjectPropertiesSelector --- .../experiment/api/AbstractRunItemImpl.java | 27 ++++++++++++------- .../experiment/api/ExpMaterialImpl.java | 7 +++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java b/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java index 6bb4fb43885..a9a94cb4034 100644 --- a/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java +++ b/experiment/src/org/labkey/experiment/api/AbstractRunItemImpl.java @@ -59,6 +59,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static java.util.Collections.emptyMap; import static org.labkey.api.util.IntegerUtils.isIntegral; /** @@ -313,16 +314,15 @@ protected List getTargetRuns(TableInfo inputTable, String rowIdColum return ExpRunImpl.fromRuns(new SqlSelector(ExperimentService.get().getSchema(), sql).getArrayList(ExperimentRun.class)); } - protected HashMap getObjectProperties(TableInfo ti) + protected Map getObjectProperties(TableInfo ti) { if (null == ti) - return new HashMap<>(); - var scope = OntologyManager.getExpSchema().getScope(); - return scope.executeWithRetryReadOnly(tx -> + return emptyMap(); + + return OntologyManager.getExpSchema().getScope().executeWithRetryReadOnly(tx -> { - var ret = new HashMap(); - var selector = new TableSelector(ti, TableSelector.ALL_COLUMNS, new SimpleFilter(FieldKey.fromParts("lsid"), getLSID()), null); - selector.forEach(rs -> + var ret = new HashMap(); + getObjectPropertiesSelector(ti).forEach(rs -> { for (ColumnInfo c : ti.getColumns()) { @@ -335,22 +335,31 @@ protected HashMap getObjectProperties(TableInfo ti) if (null != c.getMvColumnName()) { ColumnInfo mv = ti.getColumn(c.getMvColumnName()); - mvIndicator = null==mv ? null : (String)mv.getValue(rs); + mvIndicator = null == mv ? null : (String) mv.getValue(rs); } if (null == value && null == mvIndicator) continue; if (null != mvIndicator) value = null; - var prop = new ObjectProperty(getLSID(), getContainer(), c.getPropertyURI(), value, null==c.getPropertyType()? PropertyType.getFromJdbcType(c.getJdbcType()) : c.getPropertyType(), c.getName()); + + var propertyType = null == c.getPropertyType() ? PropertyType.getFromJdbcType(c.getJdbcType()) : c.getPropertyType(); + var prop = new ObjectProperty(getLSID(), getContainer(), c.getPropertyURI(), value, propertyType, c.getName()); if (null != mvIndicator) prop.setMvIndicator(mvIndicator); + ret.put(c.getPropertyURI(), prop); } }); + return ret; }); } + protected TableSelector getObjectPropertiesSelector(@NotNull TableInfo table) + { + return new TableSelector(table, new SimpleFilter(FieldKey.fromParts("lsid"), getLSID()), null); + } + protected void processIndexValues( Map props, @NotNull ExpRunItemTableImpl table, diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java index 7302e552b18..b5ada7e3c82 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java @@ -506,8 +506,7 @@ public Map getProperties(ExpSampleTypeImpl st) var ti = null == st ? null : st.getTinfo(); if (null != ti) { - var selector = new TableSelector(ti, new SimpleFilter(ExpMaterialTable.Column.RowId.fieldKey(), getRowId()), null); - selector.forEach(rs -> + getObjectPropertiesSelector(ti).forEach(rs -> { for (ColumnInfo c : ti.getColumns()) { @@ -555,9 +554,9 @@ public Map getObjectProperties(ExpSampleTypeImpl st) } @Override - public Object getProperty(DomainProperty prop) + protected TableSelector getObjectPropertiesSelector(@NotNull TableInfo table) { - return super.getProperty(prop); + return new TableSelector(table, new SimpleFilter(ExpMaterialTable.Column.RowId.fieldKey(), getRowId()), null); } @Override From 59354cb78c0fe72c2a7074c531bfba74b387b9ea Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 21 Nov 2025 11:51:32 -0800 Subject: [PATCH 05/62] nit --- .../src/org/labkey/experiment/api/ExpMaterialImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java index b5ada7e3c82..9aadd3756a7 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java @@ -245,7 +245,10 @@ public Double getAliquotVolume() } @Override - public Double getAvailableAliquotVolume() { return _object.getAvailableAliquotVolume(); } + public Double getAvailableAliquotVolume() + { + return _object.getAvailableAliquotVolume(); + } @Override public String getAliquotUnit() From 32077be79d2f91eac72a4c4cf45f3aa40884998a Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 21 Nov 2025 13:25:09 -0800 Subject: [PATCH 06/62] Initial ExpDataIterators refactor --- .../labkey/experiment/ExpDataIterators.java | 189 ++++++++++++------ 1 file changed, 124 insertions(+), 65 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index a6d3ac2f21f..25cd7da31f3 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -168,21 +168,7 @@ import static org.labkey.api.exp.api.ExperimentService.ALIASCOLUMNALIAS; import static org.labkey.api.exp.api.ExperimentService.QueryOptions.SkipBulkRemapCache; import static org.labkey.api.util.IntegerUtils.asLong; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Alias; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotedFromLSID; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotCount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AvailableAliquotVolume; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Folder; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Flag; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.MaterialSourceId; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Name; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RootMaterialRowId; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RowId; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.SampleState; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.*; import static org.labkey.api.query.AbstractQueryImportAction.configureLoader; import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.insert; import static org.labkey.experiment.api.SampleTypeUpdateServiceDI.PARENT_RECOMPUTE_NAME_COL; @@ -874,13 +860,15 @@ public DataIterator getDataIterator(DataIteratorContext context) { DataIterator pre = _pre.getDataIterator(context); if (context.getConfigParameters().containsKey(SampleTypeUpdateServiceDI.Options.SkipDerivation)) - { return pre; - } - if (context.getInsertOption() == QueryUpdateService.InsertOption.UPDATE) - return LoggingDataIterator.wrap(new ImportWithUpdateDerivationDataIterator(pre, context, _container, _user, _currentDataType, _isSample, _checkRequiredParents)); - return LoggingDataIterator.wrap(new DerivationDataIterator(pre, context, _container, _user, _currentDataType, _isSample, _skipAliquot)); + if (context.getInsertOption() != QueryUpdateService.InsertOption.UPDATE) + return LoggingDataIterator.wrap(new DerivationDataIterator(pre, context, _container, _user, _currentDataType, _isSample, _skipAliquot)); + + if (_isSample) + return LoggingDataIterator.wrap(new SampleUpdateDerivationDataIterator(pre, context, _container, _user, _currentDataType, _checkRequiredParents)); + + return LoggingDataIterator.wrap(new DataUpdateDerivationDataIterator(pre, context, _container, _user, _currentDataType, _checkRequiredParents)); } } @@ -1274,24 +1262,24 @@ else if (!_skipAliquot && _context.getInsertOption().mergeRows) } } - static class ImportWithUpdateDerivationDataIterator extends DerivationDataIteratorBase + static class SampleUpdateDerivationDataIterator extends DerivationDataIteratorBase { - // Map from Data name to Set of (parentColName, parentName) - final Map>> _parentNames; - final Integer _aliquotParentCol; - // Map of Data name and its aliquotedFromLSID - final Map _aliquotParents; - final boolean _useLsid; + final Integer _aliquotParentCol; // Map from Data name to Set of (parentColName, parentName) + final Map _aliquotParents; // Map of Data name and its aliquotedFromLSID + final Map>> _parentNames; + final Integer _rowIdCol; + final boolean _useRowId; - protected ImportWithUpdateDerivationDataIterator(DataIterator di, DataIteratorContext context, Container container, User user, ExpObject currentDataType, boolean isSample, boolean checkRequiredParent) + protected SampleUpdateDerivationDataIterator(DataIterator di, DataIteratorContext context, Container container, User user, ExpObject currentDataType, boolean checkRequiredParent) { - super(di, context, container, user, currentDataType, isSample, checkRequiredParent); + super(di, context, container, user, currentDataType, true, checkRequiredParent); Map map = DataIteratorUtil.createColumnNameMap(di); _parentNames = new LinkedHashMap<>(); _aliquotParents = new LinkedHashMap<>(); - _aliquotParentCol = isSample() ? map.getOrDefault(AliquotedFromLSID.name(), -1) : -1; - _useLsid = map.containsKey("lsid") && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); + _aliquotParentCol = map.getOrDefault(AliquotedFromLSID.name(), -1); + _rowIdCol = map.getOrDefault(RowId.name(), -1); + _useRowId = map.containsKey(RowId.name()); } @Override @@ -1306,11 +1294,17 @@ public boolean next() throws BatchValidationException // For each iteration, collect the parent col values if (hasNext) { - String key = null; - if (_useLsid && _lsidCol != null) - key = (String) get(_lsidCol); + Object key = null; + if (_useRowId && _rowIdCol != null) + { + key = get(_rowIdCol); + if (key instanceof String k) + key = Long.parseLong(k); + else + key = asLong(key); + } else if (_nameCol != null) - key = (String) get(_nameCol); + key = get(_nameCol); String aliquotParentName = null; @@ -1366,50 +1360,114 @@ else if (o instanceof Number) Map dataCache = new LongHashMap<>(); List runRecords = new ArrayList<>(); - Set keys = new LinkedHashSet<>(); + Set keys = new LinkedHashSet<>(); keys.addAll(_parentNames.keySet()); keys.addAll(_aliquotParents.keySet()); - ExperimentService experimentService = ExperimentService.get(); - for (String key : keys) + for (Object key : keys) { - Set> parentNames = _parentNames.getOrDefault(key, Collections.emptySet()); + ExpMaterial expMaterial = _useRowId ? ExperimentService.get().getExpMaterial((Long) key) : getSampleType().getSample(_container, (String) key); + if (expMaterial == null) + continue; - ExpRunItem runItem; + materialCache.put(expMaterial.getRowId(), expMaterial); + String dataType = getSampleType().getName(); String aliquotedFromLSID = _aliquotParents.get(key); - String dataType = null; - if (isSample()) - { - ExpMaterial m = _useLsid ? experimentService.getExpMaterial(key) : getSampleType().getSample(_container, key); + Set> parentNames = _parentNames.getOrDefault(key, Collections.emptySet()); - if (m != null) - { - materialCache.put(m.getRowId(), m); - dataType = getSampleType().getName(); - } - runItem = m; - } - else - { - ExpData d = _useLsid ? experimentService.getExpData(key) : getDataClass().getData(_container, key); + _processRun(expMaterial, runRecords, parentNames, cache, materialCache, dataCache, aliquotedFromLSID, dataType, true); + } - if (d != null) - { - dataCache.put(d.getRowId(), d); - dataType = getDataClass().getName(); - } - runItem = d; - } - if (runItem == null) // nothing to do if the item does not exist + if (!runRecords.isEmpty()) + ExperimentService.get().deriveSamplesBulk(runRecords, new ViewBackgroundInfo(_container, _user, null), null); + } + catch (ExperimentException e) + { + throw new RuntimeException(e); + } + catch (ValidationException e) + { + getErrors().addRowError(e); + throw getErrors(); + } + } + + return hasNext; + } + } + + static class DataUpdateDerivationDataIterator extends DerivationDataIteratorBase + { + // Map from Data name to Set of (parentColName, parentName) + final Map>> _parentNames; + final boolean _useLsid; + + protected DataUpdateDerivationDataIterator(DataIterator di, DataIteratorContext context, Container container, User user, ExpObject currentDataType, boolean checkRequiredParent) + { + super(di, context, container, user, currentDataType, false, checkRequiredParent); + + Map map = DataIteratorUtil.createColumnNameMap(di); + _parentNames = new LinkedHashMap<>(); + _useLsid = map.containsKey("lsid") && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); + } + + @Override + public boolean next() throws BatchValidationException + { + boolean hasNext = super.next(); + + // skip processing if there are errors upstream + if (getErrors().hasErrors()) + return hasNext; + + // For each iteration, collect the parent col values + if (hasNext) + { + String key = null; + if (_useLsid && _lsidCol != null) + key = (String) get(_lsidCol); + else if (_nameCol != null) + key = (String) get(_nameCol); + + for (Integer parentCol : _requiredParentCols.keySet()) + { + Object parentVal = get(parentCol); + if (parentVal == null || (parentVal instanceof String s && s.isEmpty())) + getErrors().addRowError(new ValidationException("Missing value for required property: " + _requiredParentCols.get(parentCol))); + } + + Set> allParts = _getParentParts(); + if (!allParts.isEmpty()) + _parentNames.put(key, allParts); + } + + if (getErrors().hasErrors()) + return hasNext; + + if (!hasNext) + { + try + { + RemapCache cache = new RemapCache(true); + Map materialCache = new LongHashMap<>(); + Map dataCache = new LongHashMap<>(); + + List runRecords = new ArrayList<>(); + for (String key : _parentNames.keySet()) + { + ExpData expData = _useLsid ? ExperimentService.get().getExpData(key) : getDataClass().getData(_container, key); + if (expData == null) continue; - _processRun(runItem, runRecords, parentNames, cache, materialCache, dataCache, aliquotedFromLSID, dataType, true); + dataCache.put(expData.getRowId(), expData); + String dataType = getDataClass().getName(); + Set> parentNames = _parentNames.getOrDefault(key, Collections.emptySet()); + + _processRun(expData, runRecords, parentNames, cache, materialCache, dataCache, null, dataType, true); } if (!runRecords.isEmpty()) - { ExperimentService.get().deriveSamplesBulk(runRecords, new ViewBackgroundInfo(_container, _user, null), null); - } } catch (ExperimentException e) { @@ -1421,6 +1479,7 @@ else if (o instanceof Number) throw getErrors(); } } + return hasNext; } } From f363b2cf2c6432f33f09ccc771ca2909d557d291 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 21 Nov 2025 14:56:04 -0800 Subject: [PATCH 07/62] Restore merge/update dynamic --- api/src/org/labkey/api/exp/query/ExpMaterialTable.java | 1 + .../org/labkey/experiment/api/ExpMaterialTableImpl.java | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java index 280a6ce14aa..d8f4c9af894 100644 --- a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java +++ b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java @@ -72,6 +72,7 @@ enum Column private boolean _hasUnit = false; private final String _label; + Column() { _label = ColumnInfo.labelFromName(name()); diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 865a43dda8f..6bdc56ac11d 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -1787,12 +1787,10 @@ public List getUniqueIndices() return Collections.unmodifiableList(ret); } - // // UpdatableTableInfo // - @Override public @Nullable Long getOwnerObjectId() { @@ -1820,8 +1818,11 @@ public CaseInsensitiveHashMap remapSchemaColumns() } @Override - public @NotNull Set getAltMergeKeys(DataIteratorContext context) + public Set getAltMergeKeys(DataIteratorContext context) { + if (context.getInsertOption().updateOnly) + return getAltKeysForUpdate(); + return MATERIAL_ALT_MERGE_KEYS; } From e14d648b81e2099de059e62929408c4b5633f5c9 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 21 Nov 2025 15:25:00 -0800 Subject: [PATCH 08/62] Remove LSID support allowUpdate --- .../labkey/experiment/ExpDataIterators.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 25cd7da31f3..4599c4195d1 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2327,14 +2327,10 @@ public DataIterator getDataIterator(DataIteratorContext context) dontUpdate.add("lastindexed"); } - boolean isMergeOrUpdate = context.getInsertOption().allowUpdate; - CaseInsensitiveHashSet keyColumns = new CaseInsensitiveHashSet(); CaseInsensitiveHashSet propertyKeyColumns = new CaseInsensitiveHashSet(); - if (!isMergeOrUpdate) - keyColumns.add(ExpDataTable.Column.LSID.toString()); - boolean canUpdateNames = NameExpressionOptionService.get().getAllowUserSpecificNamesValue(_container); + boolean isMergeOrUpdate = context.getInsertOption().allowUpdate; if (isSample) { @@ -2366,18 +2362,25 @@ public DataIterator getDataIterator(DataIteratorContext context) AvailableAliquotVolume.name() ); } - else if (isMergeOrUpdate) + else { - boolean isUpdateUsingLsid = context.getInsertOption().updateOnly && colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); - if (isUpdateUsingLsid) + if (isMergeOrUpdate) { - keyColumns.add(ExpDataTable.Column.LSID.name()); - if (!canUpdateNames) - dontUpdate.add("name"); + boolean isUpdateUsingLsid = context.getInsertOption().updateOnly && colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); + if (isUpdateUsingLsid) + { + keyColumns.add(ExpDataTable.Column.LSID.name()); + if (!canUpdateNames) + dontUpdate.add(ExpDataTable.Column.Name.name()); + } + else + { + keyColumns.addAll(((ExpDataClassDataTableImpl) _expTable).getAltMergeKeys(context)); + } } else { - keyColumns.addAll(((ExpDataClassDataTableImpl) _expTable).getAltMergeKeys(context)); + keyColumns.add(ExpDataTable.Column.LSID.toString()); } } From d72b527ff14a6c8d228b3115ef131320eff6e12c Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 21 Nov 2025 16:54:49 -0800 Subject: [PATCH 09/62] Merge v Update Round 12373 --- api/src/org/labkey/api/exp/query/ExpTable.java | 6 ++---- .../src/org/labkey/experiment/ExpDataIterators.java | 8 ++++++-- .../experiment/api/ExpDataClassDataTableImpl.java | 2 +- .../labkey/experiment/api/ExpMaterialTableImpl.java | 10 +++++++++- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpTable.java b/api/src/org/labkey/api/exp/query/ExpTable.java index 332eb882400..5b16a24ccb0 100644 --- a/api/src/org/labkey/api/exp/query/ExpTable.java +++ b/api/src/org/labkey/api/exp/query/ExpTable.java @@ -33,7 +33,6 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.security.permissions.Permission; -import java.util.Collections; import java.util.Set; public interface ExpTable extends ContainerFilterable, TableInfo @@ -95,7 +94,6 @@ default MutableColumnInfo addColumns(Domain domain, @Nullable String legacyName) MutableColumnInfo addColumns(Domain domain, @Nullable String legacyName,@Nullable ContainerFilter cf); - void setTitle(String title); void setDescription(String description); @@ -132,9 +130,9 @@ default ColumnInfo getExpObjectColumn() return null; } - @NotNull default Set getAltMergeKeys(DataIteratorContext context) + @Nullable default Set getAltMergeKeys(DataIteratorContext context) { - return Collections.emptySet(); + return null; } class ExpObjectDataColumn extends DataColumn diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 4599c4195d1..b8a8f686222 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2347,7 +2347,9 @@ public DataIterator getDataIterator(DataIteratorContext context) } else { - keyColumns.addAll(expMaterialTable.getAltMergeKeys(context)); + Set altMergeKeys = expMaterialTable.getAltMergeKeys(context); + if (altMergeKeys != null) + keyColumns.addAll(altMergeKeys); propertyKeyColumns.add(Name.name()); } } @@ -2375,7 +2377,9 @@ public DataIterator getDataIterator(DataIteratorContext context) } else { - keyColumns.addAll(((ExpDataClassDataTableImpl) _expTable).getAltMergeKeys(context)); + Set altMergeKeys = ((ExpDataClassDataTableImpl) _expTable).getAltMergeKeys(context); + if (altMergeKeys != null) + keyColumns.addAll(altMergeKeys); } } else diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index 8829eb21de5..60cf641b3ae 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -881,7 +881,7 @@ public CaseInsensitiveHashMap remapSchemaColumns() } @Override - public @NotNull Set getAltMergeKeys(DataIteratorContext context) + public @Nullable Set getAltMergeKeys(DataIteratorContext context) { if (context.getInsertOption().updateOnly && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) return getAltKeysForUpdate(); diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 6bdc56ac11d..67703a051c6 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -161,7 +161,7 @@ public class ExpMaterialTableImpl extends ExpRunItemTableImpl MATERIAL_ALT_MERGE_KEYS; public static final List AMOUNT_RANGE_VALIDATORS = new ArrayList<>(); static { - MATERIAL_ALT_MERGE_KEYS = Set.of(MaterialSourceId.name(), Name.name()); + MATERIAL_ALT_MERGE_KEYS = CaseInsensitiveHashSet.of(MaterialSourceId.name(), Name.name()); Lsid rangeValidatorLsid = DefaultPropertyValidator.createValidatorURI(PropertyValidatorType.Range); IPropertyValidator amountValidator = PropertyService.get().createValidator(rangeValidatorLsid.toString()); @@ -1826,6 +1826,14 @@ public Set getAltMergeKeys(DataIteratorContext context) return MATERIAL_ALT_MERGE_KEYS; } + @Override + public @NotNull Set getAltKeysForUpdate() + { + // TODO: Seems like we should not need to specify this but in some cases we skip logic because + // there are no explicit merge keys. Namely, see TriggerDataBuilderHelper.Before.getDataIterator(). + return CaseInsensitiveHashSet.of(RowId.name()); + } + @Override @NotNull public List> getAdditionalRequiredInsertColumns() From 363ed9b02ff6c782fcdc63869c1ac384b20411cb Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 25 Nov 2025 00:39:40 -0800 Subject: [PATCH 10/62] Resolve keys --- .../SampleUpdateAddColumnsDataIterator.java | 7 +-- .../labkey/experiment/ExpDataIterators.java | 28 +++++++-- .../experiment/api/ExpMaterialTableImpl.java | 15 ++--- .../api/SampleTypeUpdateServiceDI.java | 57 +++++++++++++++---- 4 files changed, 74 insertions(+), 33 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java b/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java index 3907cb8809d..56883c63969 100644 --- a/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java @@ -18,12 +18,7 @@ import java.util.Set; import java.util.function.Supplier; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.AliquotedFromLSID; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.MaterialSourceId; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Name; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RootMaterialRowId; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RowId; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.SampleState; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.*; import static org.labkey.api.util.IntegerUtils.asInteger; public class SampleUpdateAddColumnsDataIterator extends WrapperDataIterator diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index b8a8f686222..c1ba464fd12 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2338,10 +2338,26 @@ public DataIterator getDataIterator(DataIteratorContext context) if (isMergeOrUpdate) { - boolean isUpdateUsingRowId = context.getInsertOption().updateOnly && colNameMap.containsKey(RowId.name()); - if (isUpdateUsingRowId) + if (context.getInsertOption().updateOnly) { - keyColumns.add(RowId.name()); + // Both exp.Material and the provisioned tables have RowId + if (colNameMap.containsKey(RowId.name())) + keyColumns.add(RowId.name()); + else + { + // Otherwise, look for alternative keys that have been provided + for (String altKey : expMaterialTable.getAltKeysForUpdate()) + { + if (colNameMap.containsKey(altKey)) + { + keyColumns.add(altKey); + if (_propertiesTable.getColumn(altKey) != null) + propertyKeyColumns.add(altKey); + // TODO: Seems likes we should prevent update of these columns as well + } + } + } + if (!canUpdateNames) dontUpdate.add(Name.name()); } @@ -2390,7 +2406,7 @@ public DataIterator getDataIterator(DataIteratorContext context) // Since we support detailed audit logging add the ExistingRecordDataIterator here just before TableInsertDataIterator // this is a NOOP unless we are merging/updating and detailed logging is enabled - DataIteratorBuilder step2a = ExistingRecordDataIterator.createBuilder(step1, _expTable, keyColumns, Set.of(ExpMaterialTable.Column.MaterialSourceId.name(), ExpDataClassDataTable.Column.ClassId.name()), true); + DataIteratorBuilder step2a = ExistingRecordDataIterator.createBuilder(step1, _expTable, keyColumns, Set.of(ExpMaterialTable.Column.MaterialSourceId.name(), ExpDataClassDataTable.Column.ClassId.name()), !isSample); // Add RootMaterialRowId if it does not exist DataIteratorBuilder step2b = ctx -> { @@ -3070,8 +3086,8 @@ private void writeRowsToFile(TypeData typeData) if (_isSamples) { - filter = new SimpleFilter(FieldKey.fromParts("MaterialSourceId"), dataType.getRowId()); - filter.addCondition(FieldKey.fromParts("Name"), typeData.dataIds, CompareType.IN); + filter = new SimpleFilter(MaterialSourceId.fieldKey(), dataType.getRowId()); + filter.addCondition(Name.fieldKey(), typeData.dataIds, CompareType.IN); tableInfo = ExperimentService.get().getTinfoMaterial(); } else diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 67703a051c6..fed90c55d5d 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -158,10 +158,10 @@ public class ExpMaterialTableImpl extends ExpRunItemTableImpl _uniqueIdFields; boolean _supportTableRules = true; - public static final Set MATERIAL_ALT_MERGE_KEYS; - public static final List AMOUNT_RANGE_VALIDATORS = new ArrayList<>(); + private static final Set MATERIAL_ALT_KEYS; + private static final List AMOUNT_RANGE_VALIDATORS = new ArrayList<>(); static { - MATERIAL_ALT_MERGE_KEYS = CaseInsensitiveHashSet.of(MaterialSourceId.name(), Name.name()); + MATERIAL_ALT_KEYS = CaseInsensitiveHashSet.of(MaterialSourceId.name(), Name.name()); Lsid rangeValidatorLsid = DefaultPropertyValidator.createValidatorURI(PropertyValidatorType.Range); IPropertyValidator amountValidator = PropertyService.get().createValidator(rangeValidatorLsid.toString()); @@ -1820,18 +1820,13 @@ public CaseInsensitiveHashMap remapSchemaColumns() @Override public Set getAltMergeKeys(DataIteratorContext context) { - if (context.getInsertOption().updateOnly) - return getAltKeysForUpdate(); - - return MATERIAL_ALT_MERGE_KEYS; + return MATERIAL_ALT_KEYS; } @Override public @NotNull Set getAltKeysForUpdate() { - // TODO: Seems like we should not need to specify this but in some cases we skip logic because - // there are no explicit merge keys. Namely, see TriggerDataBuilderHelper.Before.getDataIterator(). - return CaseInsensitiveHashSet.of(RowId.name()); + return MATERIAL_ALT_KEYS; } @Override diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index bc90e197b94..023b2b4af0c 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -884,11 +884,12 @@ protected Map _update(User user, Container c, Map> deleteRows(User user, Container container, List for (Map k : keys) { - Long rowId = getMaterialRowId(k); // Issue 40621 // adding input fields is expensive, skip input fields for delete since deleted samples are not surfaced on Timeline UI - Map map = getMaterialMap(rowId, getMaterialLsid(k), user, container, false); + Map map = getMaterialMap(k); if (map == null) throw new QueryUpdateServiceException("No Sample Type Material found for RowID or LSID"); - if (rowId == null) - rowId = getMaterialRowId(map); + Long rowId = getMaterialRowId(map); if (rowId == null) throw new QueryUpdateServiceException("RowID is required to delete a Sample Type Material"); - Long sampleStateId = MapUtils.getLong(map,SampleState.name()); + Long sampleStateId = MapUtils.getLong(map, SampleState.name()); if (!SampleStatusService.get().isOperationPermitted(getContainer(), sampleStateId, SampleTypeService.SampleOperations.Delete)) { DataState dataState = SampleStatusService.get().getStateForRowId(container, sampleStateId); @@ -1177,8 +1176,43 @@ public List> deleteRows(User user, Container container, List return MapUtils.getLong(row, RowId.name()); } - private Map getMaterialMap(Long rowId, String lsid, User user, Container container, boolean addInputs) - throws QueryUpdateServiceException + private @Nullable Filter getMaterialFilter(Map keys) + { + Long rowId = getMaterialRowId(keys); + if (rowId != null) + return new SimpleFilter(RowId.fieldKey(), rowId); + + String lsid = getMaterialLsid(keys); + if (lsid != null) + return new SimpleFilter(LSID.fieldKey(), lsid); + + String name = getMaterialName(keys); + Integer materialSourceId = getMaterialSourceId(keys); + if (name != null && materialSourceId != null) + { + SimpleFilter filter = new SimpleFilter(Name.fieldKey(), name); + filter.addCondition(MaterialSourceId.fieldKey(), materialSourceId); + return filter; + } + + return null; + } + + private Map getMaterialMap(Map keys) throws QueryUpdateServiceException + { + Filter filter = getMaterialFilter(keys); + if (filter == null) + throw new QueryUpdateServiceException("Either RowId, LSID, or Name and MaterialSourceId is required to get Sample Type Material."); + + return new TableSelector(getQueryTable(), filter, null).getMap(); + } + + private @Nullable Map getMaterialMapWithInputs( + Long rowId, + String lsid, + User user, + Container container + ) throws QueryUpdateServiceException { Filter filter; if (rowId != null) @@ -1189,7 +1223,7 @@ else if (lsid != null) throw new QueryUpdateServiceException("Either RowId or LSID is required to get Sample Type Material."); Map sampleRow = new TableSelector(getQueryTable(), filter, null).getMap(); - if (null == sampleRow || !addInputs) + if (null == sampleRow) return sampleRow; ExperimentService experimentService = ExperimentService.get(); @@ -1451,10 +1485,11 @@ public List> getRows(User user, Container container, List> result = new ArrayList<>(keys.size()); for (Map k : keys) { - Map materialMap = getMaterialMap(getMaterialRowId(k), getMaterialLsid(k), user, container, false); + Map materialMap = getMaterialMap(k); if (materialMap != null) result.add(materialMap); } @@ -1464,7 +1499,7 @@ public List> getRows(User user, Container container, List getRow(User user, Container container, Map keys) throws QueryUpdateServiceException { - return getMaterialMap(getMaterialRowId(keys), getMaterialLsid(keys), user, container, true); + return getMaterialMapWithInputs(getMaterialRowId(keys), getMaterialLsid(keys), user, container); } private void onSamplesChanged(List> results, Map params, Container container, SampleTypeServiceImpl.SampleChangeType reason) From 62043e110748f3b7786f18e838e7baaf2efb3a4e Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 25 Nov 2025 11:11:33 -0800 Subject: [PATCH 11/62] Use getRows --- experiment/src/org/labkey/experiment/ExpDataIterators.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index c1ba464fd12..3db8c2e813c 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2342,7 +2342,10 @@ public DataIterator getDataIterator(DataIteratorContext context) { // Both exp.Material and the provisioned tables have RowId if (colNameMap.containsKey(RowId.name())) + { keyColumns.add(RowId.name()); + propertyKeyColumns.add(RowId.name()); + } else { // Otherwise, look for alternative keys that have been provided @@ -2406,7 +2409,7 @@ public DataIterator getDataIterator(DataIteratorContext context) // Since we support detailed audit logging add the ExistingRecordDataIterator here just before TableInsertDataIterator // this is a NOOP unless we are merging/updating and detailed logging is enabled - DataIteratorBuilder step2a = ExistingRecordDataIterator.createBuilder(step1, _expTable, keyColumns, Set.of(ExpMaterialTable.Column.MaterialSourceId.name(), ExpDataClassDataTable.Column.ClassId.name()), !isSample); + DataIteratorBuilder step2a = ExistingRecordDataIterator.createBuilder(step1, _expTable, keyColumns, Set.of(ExpMaterialTable.Column.MaterialSourceId.name(), ExpDataClassDataTable.Column.ClassId.name()), true); // Add RootMaterialRowId if it does not exist DataIteratorBuilder step2b = ctx -> { From 7167f3c8b42cfdd957b1db75056b7c3f9dcff742 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 25 Nov 2025 12:21:10 -0800 Subject: [PATCH 12/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index 5ee98e7f352..e1ab00f132b 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "6.72.0" + "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "6.72.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.0.tgz", - "integrity": "sha512-ouYWpvQBF0GZ/j/ErGRcAOHTAwkGP/fSA4hDKaql59U1kMGI7gZdoHZNb5aX0YWX+FLor8FDqLXz9WWmkykEWw==", + "version": "6.72.2-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.2-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-qZOOfQnFML7rkmdOHQrX1H0if3hpqZ0Yj2rEXAtzfpFgx06XUEZggF9UUKtURoxIccPjOhuy4m64BLEbT/DCaw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index 9feb45a746a..58ea6ada746 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "6.72.0" + "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index 2753d73b83a..bc4b408216e 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "6.72.0", + "@labkey/components": "6.72.2-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "6.72.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.0.tgz", - "integrity": "sha512-ouYWpvQBF0GZ/j/ErGRcAOHTAwkGP/fSA4hDKaql59U1kMGI7gZdoHZNb5aX0YWX+FLor8FDqLXz9WWmkykEWw==", + "version": "6.72.2-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.2-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-qZOOfQnFML7rkmdOHQrX1H0if3hpqZ0Yj2rEXAtzfpFgx06XUEZggF9UUKtURoxIccPjOhuy4m64BLEbT/DCaw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 9e7daa9b776..a77037c1c10 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "6.72.0", + "@labkey/components": "6.72.2-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index af738667dc1..f328ceb2ab1 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "6.72.0" + "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "6.72.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.0.tgz", - "integrity": "sha512-ouYWpvQBF0GZ/j/ErGRcAOHTAwkGP/fSA4hDKaql59U1kMGI7gZdoHZNb5aX0YWX+FLor8FDqLXz9WWmkykEWw==", + "version": "6.72.2-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.2-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-qZOOfQnFML7rkmdOHQrX1H0if3hpqZ0Yj2rEXAtzfpFgx06XUEZggF9UUKtURoxIccPjOhuy4m64BLEbT/DCaw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index 00d4a9b2001..f93eb678aad 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "6.72.0" + "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index 6a6710ddab5..c3d92aa2aa1 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "6.72.0" + "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "6.72.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.0.tgz", - "integrity": "sha512-ouYWpvQBF0GZ/j/ErGRcAOHTAwkGP/fSA4hDKaql59U1kMGI7gZdoHZNb5aX0YWX+FLor8FDqLXz9WWmkykEWw==", + "version": "6.72.2-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.2-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-qZOOfQnFML7rkmdOHQrX1H0if3hpqZ0Yj2rEXAtzfpFgx06XUEZggF9UUKtURoxIccPjOhuy4m64BLEbT/DCaw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index fa8766f4f9f..99684a98afb 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "6.72.0" + "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", From ac7ba3a84fb9fae3d0e17d0ddd71c4ba459d67c4 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 25 Nov 2025 12:43:26 -0800 Subject: [PATCH 13/62] ExistingRecordDataIterator: check for key columns rather than requiring all specified - always include table key columns --- .../ExistingRecordDataIterator.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java b/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java index 17d143deb40..a73ba0b77b9 100644 --- a/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java @@ -32,7 +32,6 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -95,15 +94,21 @@ public abstract class ExistingRecordDataIterator extends WrapperDataIterator var map = DataIteratorUtil.createColumnNameMap(in); containerCol = map.get("Container"); - Collection keyNames = null==keys ? target.getPkColumnNames() : keys; + Set keyColumnNames = new CaseInsensitiveHashSet(target.getPkColumnNames()); + if (keys != null) + keyColumnNames.addAll(keys); if (sharedKeys != null) _sharedKeys.addAll(sharedKeys); - _dataColumnNames.addAll(detailed ? map.keySet() : keyNames); + if (detailed) + _dataColumnNames.addAll(map.keySet()); - for (String name : keyNames) + for (String name : keyColumnNames) { + if (!map.containsKey(name)) + continue; + Integer index = map.get(name); ColumnInfo col = target.getColumn(name); if (null == index || null == col) @@ -114,7 +119,11 @@ public abstract class ExistingRecordDataIterator extends WrapperDataIterator } pkSuppliers.add(in.getSupplier(index)); pkColumns.add(col); + _dataColumnNames.add(name); } + + if (pkColumns.isEmpty()) + throw new IllegalArgumentException("At least one key column is required."); } @Override From 7ccfa3788c5336bea86474d8f86e452e71d3948b Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 25 Nov 2025 12:44:37 -0800 Subject: [PATCH 14/62] comment --- experiment/src/org/labkey/experiment/ExpDataIterators.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 3db8c2e813c..c2625626a9d 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2356,7 +2356,7 @@ public DataIterator getDataIterator(DataIteratorContext context) keyColumns.add(altKey); if (_propertiesTable.getColumn(altKey) != null) propertyKeyColumns.add(altKey); - // TODO: Seems likes we should prevent update of these columns as well + // TODO: Should we prevent update of these columns when they are being used as a key column? } } } From fcd6963fdd990f59a99d7d784c1bfd117dbc2bf8 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 25 Nov 2025 16:32:44 -0800 Subject: [PATCH 15/62] Update validation --- .../api/data/validator/RequiredValidator.java | 11 ++ .../labkey/experiment/ExpDataIterators.java | 122 ++++++++++++------ .../api/SampleTypeUpdateServiceDI.java | 3 +- 3 files changed, 93 insertions(+), 43 deletions(-) diff --git a/api/src/org/labkey/api/data/validator/RequiredValidator.java b/api/src/org/labkey/api/data/validator/RequiredValidator.java index 5c26a25fe9a..60fde95dc85 100644 --- a/api/src/org/labkey/api/data/validator/RequiredValidator.java +++ b/api/src/org/labkey/api/data/validator/RequiredValidator.java @@ -15,6 +15,7 @@ */ package org.labkey.api.data.validator; +import org.jetbrains.annotations.Nullable; import org.labkey.api.exp.MvFieldWrapper; /** @@ -26,12 +27,19 @@ public class RequiredValidator extends AbstractColumnValidator implements Unders { final boolean allowMV; final boolean allowES; + final String _message; public RequiredValidator(String columnName, boolean allowMissingValueIndicators, boolean allowEmptyString) + { + this(columnName, allowMissingValueIndicators, allowEmptyString, null); + } + + public RequiredValidator(String columnName, boolean allowMissingValueIndicators, boolean allowEmptyString, @Nullable String message) { super(columnName); allowMV = allowMissingValueIndicators; allowES = allowEmptyString; + _message = message; } @Override @@ -59,6 +67,9 @@ protected String _validate(int rowNum, Object value) return null; } + if (_message != null) + return _message; + // DatasetDefinition.importDatasetData:: errors.add("Row " + rowNumber + " does not contain required field " + col.getName() + "."); // OntologyManager.insertTabDelimited:: throw new ValidationException("Missing value for required property " + col.getName()); return "Missing value for required property: " + _columnName; diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index c2625626a9d..979efb97d30 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2312,15 +2312,16 @@ public DataIterator getDataIterator(DataIteratorContext context) assert _expTable instanceof ExpMaterialTableImpl || _expTable instanceof ExpDataClassDataTableImpl; boolean isSample = _expTable instanceof ExpMaterialTableImpl; + boolean isMergeOrUpdate = context.getInsertOption().allowUpdate; + boolean isUpdateOnly = context.getInsertOption().updateOnly; SimpleTranslator step1 = new SimpleTranslator(input, context); step1.selectAll(Sets.newCaseInsensitiveHashSet(Alias.name()), _importAliases); if (colNameMap.containsKey(Alias.name())) step1.addColumn(ExperimentService.ALIASCOLUMNALIAS, colNameMap.get(Alias.name())); // see AliasDataIteratorBuilder - CaseInsensitiveHashSet dontUpdate = new CaseInsensitiveHashSet(); - dontUpdate.addAll(NOT_FOR_UPDATE); - if (context.getInsertOption().updateOnly) + CaseInsensitiveHashSet dontUpdate = new CaseInsensitiveHashSet(NOT_FOR_UPDATE); + if (isUpdateOnly) { dontUpdate.add("objectid"); dontUpdate.add("cpastype"); @@ -2330,7 +2331,6 @@ public DataIterator getDataIterator(DataIteratorContext context) CaseInsensitiveHashSet keyColumns = new CaseInsensitiveHashSet(); CaseInsensitiveHashSet propertyKeyColumns = new CaseInsensitiveHashSet(); boolean canUpdateNames = NameExpressionOptionService.get().getAllowUserSpecificNamesValue(_container); - boolean isMergeOrUpdate = context.getInsertOption().allowUpdate; if (isSample) { @@ -2338,7 +2338,7 @@ public DataIterator getDataIterator(DataIteratorContext context) if (isMergeOrUpdate) { - if (context.getInsertOption().updateOnly) + if (isUpdateOnly) { // Both exp.Material and the provisioned tables have RowId if (colNameMap.containsKey(RowId.name())) @@ -2387,7 +2387,7 @@ public DataIterator getDataIterator(DataIteratorContext context) { if (isMergeOrUpdate) { - boolean isUpdateUsingLsid = context.getInsertOption().updateOnly && colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); + boolean isUpdateUsingLsid = isUpdateOnly && colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); if (isUpdateUsingLsid) { keyColumns.add(ExpDataTable.Column.LSID.name()); @@ -2407,31 +2407,25 @@ public DataIterator getDataIterator(DataIteratorContext context) } } - // Since we support detailed audit logging add the ExistingRecordDataIterator here just before TableInsertDataIterator - // this is a NOOP unless we are merging/updating and detailed logging is enabled - DataIteratorBuilder step2a = ExistingRecordDataIterator.createBuilder(step1, _expTable, keyColumns, Set.of(ExpMaterialTable.Column.MaterialSourceId.name(), ExpDataClassDataTable.Column.ClassId.name()), true); - - // Add RootMaterialRowId if it does not exist - DataIteratorBuilder step2b = ctx -> { - DataIterator in = step2a.getDataIterator(ctx); - var map = DataIteratorUtil.createColumnNameMap(in); - if (map.containsKey(RootMaterialRowId.toString()) || !map.containsKey(RowId.toString())) - return in; - var ret = new SimpleTranslator(in, ctx); - ret.selectAll(); - ret.addAliasColumn(RootMaterialRowId.toString(), map.get(RowId.toString())); - return ret; - }; + // Since we support detailed audit logging, add the ExistingRecordDataIterator here just before TableInsertDataIterator. + // This is a NOOP unless we are merging/updating and detailed logging is enabled + DataIteratorBuilder dib = ExistingRecordDataIterator.createBuilder(step1, _expTable, keyColumns, Set.of(ExpMaterialTable.Column.MaterialSourceId.name(), ExpDataClassDataTable.Column.ClassId.name()), true); - DataIteratorBuilder step2c = step2b; - if (isSample && isMergeOrUpdate) + if (isSample) { - step2c = LoggingDataIterator.wrap(new ExpDataIterators.SampleStatusCheckIteratorBuilder(step2b, _container)); + // Add RootMaterialRowId if it does not exist + dib = getRootMaterialRowIdBuilder(dib); + + if (isMergeOrUpdate) + dib = new SampleStatusCheckIteratorBuilder(dib, _container); + + if (isUpdateOnly) + dib = new SampleUpdateOnlyDataIteratorBuilder(dib, context, _container, _user); } // Insert into exp.data then the provisioned table - // Use embargo data iterator to ensure rows are committed before being sent along Issue 26082 (row at a time, reselect rowid) - DataIteratorBuilder step3 = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(step2c, _expTable, _container) + // Use embargo data iterator to ensure rows are committed before being sent along Issue 26082 (row at a time, reselect rowId) + dib = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(dib, _expTable, _container) .setKeyColumns(keyColumns) .setDontUpdate(dontUpdate) .setAddlSkipColumns(_excludedColumns) @@ -2439,33 +2433,79 @@ public DataIterator getDataIterator(DataIteratorContext context) .setFailOnEmptyUpdate(false)); // pass in remap columns to help reconcile columns that may be aliased in the virtual table - DataIteratorBuilder step4 = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(step3, _propertiesTable, _container) + dib = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(dib, _propertiesTable, _container) .setKeyColumns(propertyKeyColumns.isEmpty() ? keyColumns : propertyKeyColumns) .setDontUpdate(dontUpdate) .setVocabularyProperties(PropertyService.get().findVocabularyProperties(_container, colNameMap.keySet())) - .setRemapSchemaColumns(((UpdateableTableInfo)_expTable).remapSchemaColumns()) + .setRemapSchemaColumns(((UpdateableTableInfo) _expTable).remapSchemaColumns()) .setFailOnEmptyUpdate(false)); - DataIteratorBuilder step5 = step4; if (colNameMap.containsKey(Flag.name()) || colNameMap.containsKey("comment")) - { - step5 = LoggingDataIterator.wrap(new ExpDataIterators.FlagDataIteratorBuilder(step4, _user, isSample, _dataTypeObject, _container)); - } + dib = new FlagDataIteratorBuilder(dib, _user, isSample, _dataTypeObject, _container); // Wire up derived parent/child data and materials - DataIteratorBuilder step6 = LoggingDataIterator.wrap(new ExpDataIterators.DerivationDataIteratorBuilder(step5, _container, _user, isSample, _dataTypeObject, false, false/*Validation already done in StandardDataIterator*/)); + dib = new DerivationDataIteratorBuilder(dib, _container, _user, isSample, _dataTypeObject, false, false /* Validation already done in StandardDataIterator */); - DataIteratorBuilder step7 = step6; - boolean hasRollUpColumns = colNameMap.containsKey(ROOT_RECOMPUTE_ROWID_COL); - if (isSample && !context.getConfigParameterBoolean(SampleTypeService.ConfigParameters.DeferAliquotRuns) && hasRollUpColumns) - step7 = LoggingDataIterator.wrap(new ExpDataIterators.AliquotRollupDataIteratorBuilder(step6, _container)); + if (isSample && !context.getConfigParameterBoolean(SampleTypeService.ConfigParameters.DeferAliquotRuns) && colNameMap.containsKey(ROOT_RECOMPUTE_ROWID_COL)) + dib = new AliquotRollupDataIteratorBuilder(dib, _container); // Hack: add the alias and lsid values back into the input, so we can process them in the chained data iterator - DataIteratorBuilder step8 = step7; if (null != _indexFunction) - step8 = LoggingDataIterator.wrap(new ExpDataIterators.SearchIndexIteratorBuilder(step7, _indexFunction)); // may need to add this after the aliases are set + dib = new SearchIndexIteratorBuilder(dib, _indexFunction); // may need to add this after the aliases are set + + return dib.getDataIterator(context); + } + + private DataIteratorBuilder getRootMaterialRowIdBuilder(DataIteratorBuilder dib) + { + return ctx -> { + DataIterator in = dib.getDataIterator(ctx); + var map = DataIteratorUtil.createColumnNameMap(in); + if (map.containsKey(RootMaterialRowId.toString()) || !map.containsKey(RowId.toString())) + return in; + var ret = new SimpleTranslator(in, ctx); + ret.selectAll(); + ret.addAliasColumn(RootMaterialRowId.toString(), map.get(RowId.toString())); + return ret; + }; + } + } + + private static class SampleUpdateOnlyDataIteratorBuilder implements DataIteratorBuilder + { + private final Container _container; + private final DataIteratorBuilder _in; + private final User _user; + + public SampleUpdateOnlyDataIteratorBuilder(@NotNull DataIteratorBuilder in, DataIteratorContext context, Container container, User user) + { + _container = container; + _in = in; + _user = user; + + assert context.getInsertOption().updateOnly : "SampleUpdateOnlyDataIteratorBuilder should only be used for UPDATE_ONLY"; + } + + @Override + public DataIterator getDataIterator(DataIteratorContext context) + { + DataIterator di = _in.getDataIterator(context); + ValidatorIterator validate = new ValidatorIterator(di, context, _container, _user); + Map map = DataIteratorUtil.createColumnNameMap(validate); + + Integer index = map.get(Name.name()); + if (index != null) + { + ColumnInfo column = di.getColumnInfo(index); + validate.addValidator(index, new RequiredValidator(column.getColumnName(), false, false, "Sample name cannot be blank")); + } + + // Add other column validators here... + + if (!validate.hasValidators()) + return di; - return LoggingDataIterator.wrap(step8.getDataIterator(context)); + return LoggingDataIterator.wrap(validate); } } @@ -3231,7 +3271,7 @@ public DataIterator getDataIterator(DataIteratorContext context) } } - public static class SampleStatusCheckDataIterator extends WrapperDataIterator + private static class SampleStatusCheckDataIterator extends WrapperDataIterator { private final Set SAMPLE_IMPORT_BASE_FIELDS = new CaseInsensitiveHashSet( "LSID", diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 023b2b4af0c..326d90eef92 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -888,8 +888,7 @@ protected Map _update(User user, Container c, Map Date: Wed, 26 Nov 2025 11:25:52 -0800 Subject: [PATCH 16/62] Consistent keys for ExistingRecordDataIterator --- .../ExistingRecordDataIterator.java | 10 ++- .../TriggerDataBuilderHelper.java | 50 ++++++++----- .../org/labkey/api/exp/query/ExpTable.java | 17 +++++ .../labkey/experiment/ExpDataIterators.java | 74 +++++-------------- .../api/ExpDataClassDataTableImpl.java | 31 ++++++++ .../experiment/api/ExpMaterialTableImpl.java | 38 ++++++++++ 6 files changed, 140 insertions(+), 80 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java b/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java index a73ba0b77b9..c39acf459a1 100644 --- a/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/ExistingRecordDataIterator.java @@ -94,9 +94,11 @@ public abstract class ExistingRecordDataIterator extends WrapperDataIterator var map = DataIteratorUtil.createColumnNameMap(in); containerCol = map.get("Container"); - Set keyColumnNames = new CaseInsensitiveHashSet(target.getPkColumnNames()); - if (keys != null) - keyColumnNames.addAll(keys); + Set keyNames = new CaseInsensitiveHashSet(); + if (keys == null) + keyNames.addAll(target.getPkColumnNames()); + else + keyNames.addAll(keys); if (sharedKeys != null) _sharedKeys.addAll(sharedKeys); @@ -104,7 +106,7 @@ public abstract class ExistingRecordDataIterator extends WrapperDataIterator if (detailed) _dataColumnNames.addAll(map.keySet()); - for (String name : keyColumnNames) + for (String name : keyNames) { if (!map.containsKey(name)) continue; diff --git a/api/src/org/labkey/api/dataiterator/TriggerDataBuilderHelper.java b/api/src/org/labkey/api/dataiterator/TriggerDataBuilderHelper.java index ba497930564..3f507e6a574 100644 --- a/api/src/org/labkey/api/dataiterator/TriggerDataBuilderHelper.java +++ b/api/src/org/labkey/api/dataiterator/TriggerDataBuilderHelper.java @@ -32,11 +32,6 @@ import static org.labkey.api.admin.FolderImportContext.IS_NEW_FOLDER_IMPORT_KEY; import static org.labkey.api.util.IntegerUtils.asInteger; -/** - * User: matthewb - * Date: 2011-09-07 - * Time: 5:14 PM - */ public class TriggerDataBuilderHelper { final Container _c; @@ -129,14 +124,21 @@ class Before implements DataIteratorBuilder @Override public DataIterator getDataIterator(DataIteratorContext context) { - DataIterator pre = _pre.getDataIterator(context); + DataIterator di = _pre.getDataIterator(context); if (!_target.hasTriggers(_c)) - return pre; - pre = LoggingDataIterator.wrap(pre); + return di; + di = LoggingDataIterator.wrap(di); - Set mergeKeys = null; - if (_target instanceof ExpTable) - mergeKeys = ((ExpTable)_target).getAltMergeKeys(context); + Set existingRecordKeyColumnNames = null; + Set sharedKeys = null; + boolean isMergeOrUpdate = context.getInsertOption().allowUpdate; + + if (isMergeOrUpdate && _target instanceof ExpTable expTable) + { + Map colNameMap = DataIteratorUtil.createColumnNameMap(di); + existingRecordKeyColumnNames = expTable.getExistingRecordKeyColumnNames(context, colNameMap); + sharedKeys = expTable.getExistingRecordSharedKeyColumnNames(); + } boolean isNewFolderImport = false; if (_extraContext != null && _extraContext.get(IS_NEW_FOLDER_IMPORT_KEY) != null) @@ -144,18 +146,26 @@ public DataIterator getDataIterator(DataIteratorContext context) isNewFolderImport = (boolean) _extraContext.get(IS_NEW_FOLDER_IMPORT_KEY); } - boolean skipExistingRecord = !context.getInsertOption().allowUpdate || mergeKeys == null || isNewFolderImport; - DataIterator coerce = new CoerceDataIterator(pre, context, _target, !context.getInsertOption().updateOnly); + di = LoggingDataIterator.wrap(new CoerceDataIterator(di, context, _target, !context.getInsertOption().updateOnly)); context.setWithLookupRemapping(false); - coerce = LoggingDataIterator.wrap(coerce); - if (skipExistingRecord) - return LoggingDataIterator.wrap(new BeforeIterator(new CachingDataIterator(coerce), context)); - else if (context.getInsertOption().mergeRows && !_target.supportsInsertOption(QueryUpdateService.InsertOption.MERGE)) - return LoggingDataIterator.wrap(new BeforeIterator(coerce, context)); + boolean shouldCache = true; + boolean skipExistingRecord = !isMergeOrUpdate || existingRecordKeyColumnNames == null || isNewFolderImport; + if (!skipExistingRecord) + { + if (context.getInsertOption().mergeRows) + { + if (_target.supportsInsertOption(QueryUpdateService.InsertOption.MERGE)) + di = ExistingRecordDataIterator.createBuilder(di, _target, existingRecordKeyColumnNames, sharedKeys, true).getDataIterator(context); + else + shouldCache = false; + } + } + + if (shouldCache) + di = new CachingDataIterator(di); - coerce = ExistingRecordDataIterator.createBuilder(coerce, _target, mergeKeys, null, true).getDataIterator(context); - return LoggingDataIterator.wrap(new BeforeIterator(new CachingDataIterator(coerce), context)); + return LoggingDataIterator.wrap(new BeforeIterator(di, context)); } } diff --git a/api/src/org/labkey/api/exp/query/ExpTable.java b/api/src/org/labkey/api/exp/query/ExpTable.java index 5b16a24ccb0..cfa4cab2b93 100644 --- a/api/src/org/labkey/api/exp/query/ExpTable.java +++ b/api/src/org/labkey/api/exp/query/ExpTable.java @@ -33,6 +33,7 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.security.permissions.Permission; +import java.util.Map; import java.util.Set; public interface ExpTable extends ContainerFilterable, TableInfo @@ -135,6 +136,22 @@ default ColumnInfo getExpObjectColumn() return null; } + /** + * Returns the set of key column names for this table to be specified as key columns for the ExistingRecordDataIterator. + */ + @Nullable default Set getExistingRecordKeyColumnNames(DataIteratorContext context, Map colNameMap) + { + return null; + } + + /** + * Returns the set of key column names for this table to be specified as shared key columns for the ExistingRecordDataIterator. + */ + @Nullable default Set getExistingRecordSharedKeyColumnNames() + { + return null; + } + class ExpObjectDataColumn extends DataColumn { public ExpObjectDataColumn(ColumnInfo colInfo) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 979efb97d30..8bd6e1690ef 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -85,10 +85,9 @@ import org.labkey.api.exp.property.PropertyService; import org.labkey.api.exp.query.AbstractExpSchema; import org.labkey.api.exp.query.DataClassUserSchema; -import org.labkey.api.exp.query.ExpDataClassDataTable; import org.labkey.api.exp.query.ExpDataTable; -import org.labkey.api.exp.query.ExpMaterialTable; import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.ExpTable; import org.labkey.api.exp.query.SamplesSchema; import org.labkey.api.qc.DataState; import org.labkey.api.qc.SampleStatusService; @@ -2250,7 +2249,7 @@ public boolean next() throws BatchValidationException public static class PersistDataIteratorBuilder implements DataIteratorBuilder { private final DataIteratorBuilder _in; - private final TableInfo _expTable; + private final ExpTable _expTable; private final TableInfo _propertiesTable; private final ExpObject _dataTypeObject; private final Container _container; @@ -2262,7 +2261,7 @@ public static class PersistDataIteratorBuilder implements DataIteratorBuilder final Map _importAliases; // expTable is the shared experiment table e.g. exp.Data or exp.Materials - public PersistDataIteratorBuilder(@NotNull DataIteratorBuilder in, TableInfo expTable, TableInfo propsTable, ExpObject typeObject, Container container, User user, Map importAliases) + public PersistDataIteratorBuilder(@NotNull DataIteratorBuilder in, ExpTable expTable, TableInfo propsTable, ExpObject typeObject, Container container, User user, Map importAliases) { _in = in; _expTable = expTable; @@ -2332,48 +2331,22 @@ public DataIterator getDataIterator(DataIteratorContext context) CaseInsensitiveHashSet propertyKeyColumns = new CaseInsensitiveHashSet(); boolean canUpdateNames = NameExpressionOptionService.get().getAllowUserSpecificNamesValue(_container); - if (isSample) - { - ExpMaterialTableImpl expMaterialTable = (ExpMaterialTableImpl) _expTable; + var keys = _expTable.getExistingRecordKeyColumnNames(context, colNameMap); + if (keys != null) + keyColumns.addAll(keys); - if (isMergeOrUpdate) - { - if (isUpdateOnly) - { - // Both exp.Material and the provisioned tables have RowId - if (colNameMap.containsKey(RowId.name())) - { - keyColumns.add(RowId.name()); - propertyKeyColumns.add(RowId.name()); - } - else - { - // Otherwise, look for alternative keys that have been provided - for (String altKey : expMaterialTable.getAltKeysForUpdate()) - { - if (colNameMap.containsKey(altKey)) - { - keyColumns.add(altKey); - if (_propertiesTable.getColumn(altKey) != null) - propertyKeyColumns.add(altKey); - // TODO: Should we prevent update of these columns when they are being used as a key column? - } - } - } + for (String key : keyColumns) + { + if (_propertiesTable.getColumn(key) != null) + propertyKeyColumns.add(key); + } - if (!canUpdateNames) - dontUpdate.add(Name.name()); - } - else - { - Set altMergeKeys = expMaterialTable.getAltMergeKeys(context); - if (altMergeKeys != null) - keyColumns.addAll(altMergeKeys); - propertyKeyColumns.add(Name.name()); - } - } + if (isSample) + { + if (isUpdateOnly && !canUpdateNames) + dontUpdate.add(Name.name()); - dontUpdate.addAll(expMaterialTable.getUniqueIdFields()); + dontUpdate.addAll(((ExpMaterialTableImpl) _expTable).getUniqueIdFields()); dontUpdate.addAll( RootMaterialRowId.name(), AliquotedFromLSID.name(), @@ -2390,26 +2363,15 @@ public DataIterator getDataIterator(DataIteratorContext context) boolean isUpdateUsingLsid = isUpdateOnly && colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); if (isUpdateUsingLsid) { - keyColumns.add(ExpDataTable.Column.LSID.name()); if (!canUpdateNames) dontUpdate.add(ExpDataTable.Column.Name.name()); } - else - { - Set altMergeKeys = ((ExpDataClassDataTableImpl) _expTable).getAltMergeKeys(context); - if (altMergeKeys != null) - keyColumns.addAll(altMergeKeys); - } - } - else - { - keyColumns.add(ExpDataTable.Column.LSID.toString()); } } // Since we support detailed audit logging, add the ExistingRecordDataIterator here just before TableInsertDataIterator. // This is a NOOP unless we are merging/updating and detailed logging is enabled - DataIteratorBuilder dib = ExistingRecordDataIterator.createBuilder(step1, _expTable, keyColumns, Set.of(ExpMaterialTable.Column.MaterialSourceId.name(), ExpDataClassDataTable.Column.ClassId.name()), true); + DataIteratorBuilder dib = ExistingRecordDataIterator.createBuilder(step1, _expTable, keyColumns, _expTable.getExistingRecordSharedKeyColumnNames(), true); if (isSample) { @@ -2434,7 +2396,7 @@ public DataIterator getDataIterator(DataIteratorContext context) // pass in remap columns to help reconcile columns that may be aliased in the virtual table dib = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(dib, _propertiesTable, _container) - .setKeyColumns(propertyKeyColumns.isEmpty() ? keyColumns : propertyKeyColumns) + .setKeyColumns(propertyKeyColumns) .setDontUpdate(dontUpdate) .setVocabularyProperties(PropertyService.get().findVocabularyProperties(_container, colNameMap.keySet())) .setRemapSchemaColumns(((UpdateableTableInfo) _expTable).remapSchemaColumns()) diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index 60cf641b3ae..40499f8d336 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -895,6 +895,37 @@ public Set getAltKeysForUpdate() return DATA_CLASS_ALT_UPDATE_KEYS; } + @Override + public @Nullable Set getExistingRecordKeyColumnNames(DataIteratorContext context, Map colNameMap) + { + Set keyColumnNames = new CaseInsensitiveHashSet(); + + if (context.getInsertOption().allowUpdate) + { + boolean isUpdateUsingLsid = context.getInsertOption().updateOnly && colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); + if (isUpdateUsingLsid) + keyColumnNames.add(Column.LSID.name()); + else + { + Set altMergeKeys = getAltMergeKeys(context); + if (altMergeKeys == null) + return null; + + keyColumnNames.addAll(altMergeKeys); + } + } + else + keyColumnNames.add(Column.LSID.name()); + + return keyColumnNames; + } + + @Override + public @Nullable Set getExistingRecordSharedKeyColumnNames() + { + return CaseInsensitiveHashSet.of(Column.ClassId.name()); + } + @Override @NotNull public List> getAdditionalRequiredInsertColumns() diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index fed90c55d5d..2facc062a1b 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -1829,6 +1829,44 @@ public Set getAltMergeKeys(DataIteratorContext context) return MATERIAL_ALT_KEYS; } + @Override + public @Nullable Set getExistingRecordKeyColumnNames(DataIteratorContext context, Map colNameMap) + { + if (!context.getInsertOption().allowUpdate) + return null; + + Set keyColumnNames = new CaseInsensitiveHashSet(); + if (context.getInsertOption().updateOnly) + { + if (colNameMap.containsKey(RowId.name())) + keyColumnNames.add(RowId.name()); + else + { + for (String altKey : getAltKeysForUpdate()) + { + if (colNameMap.containsKey(altKey)) + keyColumnNames.add(altKey); + } + } + } + else + { + Set altMergeKeys = getAltMergeKeys(context); + if (altMergeKeys == null) + return null; + + keyColumnNames.addAll(altMergeKeys); + } + + return keyColumnNames; + } + + @Override + public @Nullable Set getExistingRecordSharedKeyColumnNames() + { + return CaseInsensitiveHashSet.of(MaterialSourceId.name()); + } + @Override @NotNull public List> getAdditionalRequiredInsertColumns() From 43b51ee332f3b3fa78f515d229dfe62d87dbf352 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 26 Nov 2025 14:08:47 -0800 Subject: [PATCH 17/62] Revise TriggerDataBuilderHelper --- .../TriggerDataBuilderHelper.java | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/TriggerDataBuilderHelper.java b/api/src/org/labkey/api/dataiterator/TriggerDataBuilderHelper.java index 3f507e6a574..7712726b939 100644 --- a/api/src/org/labkey/api/dataiterator/TriggerDataBuilderHelper.java +++ b/api/src/org/labkey/api/dataiterator/TriggerDataBuilderHelper.java @@ -149,23 +149,16 @@ public DataIterator getDataIterator(DataIteratorContext context) di = LoggingDataIterator.wrap(new CoerceDataIterator(di, context, _target, !context.getInsertOption().updateOnly)); context.setWithLookupRemapping(false); - boolean shouldCache = true; - boolean skipExistingRecord = !isMergeOrUpdate || existingRecordKeyColumnNames == null || isNewFolderImport; - if (!skipExistingRecord) - { - if (context.getInsertOption().mergeRows) - { - if (_target.supportsInsertOption(QueryUpdateService.InsertOption.MERGE)) - di = ExistingRecordDataIterator.createBuilder(di, _target, existingRecordKeyColumnNames, sharedKeys, true).getDataIterator(context); - else - shouldCache = false; - } - } + // Skip existing records + if (!context.getInsertOption().allowUpdate || existingRecordKeyColumnNames == null || isNewFolderImport) + return LoggingDataIterator.wrap(new BeforeIterator(new CachingDataIterator(di), context)); - if (shouldCache) - di = new CachingDataIterator(di); + // Merge request but merge is not supported + if (context.getInsertOption().mergeRows && !_target.supportsInsertOption(QueryUpdateService.InsertOption.MERGE)) + return LoggingDataIterator.wrap(new BeforeIterator(di, context)); - return LoggingDataIterator.wrap(new BeforeIterator(di, context)); + di = ExistingRecordDataIterator.createBuilder(di, _target, existingRecordKeyColumnNames, sharedKeys, true).getDataIterator(context); + return LoggingDataIterator.wrap(new BeforeIterator(new CachingDataIterator(di), context)); } } From ac6fa88109185995bd41b975a7068eb4fd2c8b26 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 26 Nov 2025 14:16:17 -0800 Subject: [PATCH 18/62] logic --- .../org/labkey/experiment/ExpDataIterators.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 8bd6e1690ef..5a1866822c7 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -857,17 +857,18 @@ public DerivationDataIteratorBuilder(DataIteratorBuilder pre, Container containe @Override public DataIterator getDataIterator(DataIteratorContext context) { - DataIterator pre = _pre.getDataIterator(context); + DataIterator di = _pre.getDataIterator(context); if (context.getConfigParameters().containsKey(SampleTypeUpdateServiceDI.Options.SkipDerivation)) - return pre; + return di; if (context.getInsertOption() != QueryUpdateService.InsertOption.UPDATE) - return LoggingDataIterator.wrap(new DerivationDataIterator(pre, context, _container, _user, _currentDataType, _isSample, _skipAliquot)); - - if (_isSample) - return LoggingDataIterator.wrap(new SampleUpdateDerivationDataIterator(pre, context, _container, _user, _currentDataType, _checkRequiredParents)); + di = new DerivationDataIterator(di, context, _container, _user, _currentDataType, _isSample, _skipAliquot); + else if (_isSample) + di = new SampleUpdateDerivationDataIterator(di, context, _container, _user, _currentDataType, _checkRequiredParents); + else + di = new DataUpdateDerivationDataIterator(di, context, _container, _user, _currentDataType, _checkRequiredParents); - return LoggingDataIterator.wrap(new DataUpdateDerivationDataIterator(pre, context, _container, _user, _currentDataType, _checkRequiredParents)); + return LoggingDataIterator.wrap(di); } } From 5fc9dfccd21cb7f18598832a5a3a25cb26c8973b Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 26 Nov 2025 15:11:10 -0800 Subject: [PATCH 19/62] SampleUpdateNamePolicyDataIterator --- .../labkey/experiment/ExpDataIterators.java | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 5a1866822c7..7aa150e3ed1 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2465,10 +2465,47 @@ public DataIterator getDataIterator(DataIteratorContext context) // Add other column validators here... - if (!validate.hasValidators()) - return di; + if (validate.hasValidators()) + di = validate; + + return LoggingDataIterator.wrap(new SampleUpdateNamePolicyDataIterator(di, context, _container)); + } + } + + private static class SampleUpdateNamePolicyDataIterator extends WrapperDataIterator + { + private final DataIteratorContext _context; + private final Integer _nameCol; + + protected SampleUpdateNamePolicyDataIterator(DataIterator di, DataIteratorContext context, Container container) + { + super(di); + _context = context; + + if (NameExpressionOptionService.get().getAllowUserSpecificNamesValue(container)) + _nameCol = null; + else + _nameCol = DataIteratorUtil.createColumnNameMap(di).get(Name.name()); + } - return LoggingDataIterator.wrap(validate); + @Override + public boolean next() throws BatchValidationException + { + boolean hasNext = super.next(); + if (!hasNext) + return false; + + if (_nameCol == null || _context.getErrors().hasErrors() || getExistingRecord() == null) + return true; + + Object newNameObj = get(_nameCol); + String newName = newNameObj == null ? null : String.valueOf(newNameObj); + String oldName = (String) getExistingRecord().get(Name.name()); + + if (!StringUtils.isEmpty(newName) && !newName.equals(oldName)) + _context.getErrors().addRowError(new ValidationException("User-specified sample name not allowed")); + + return true; } } From 9828b4b70723ce8655a0429397d38bff09e4b363 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 26 Nov 2025 16:08:33 -0800 Subject: [PATCH 20/62] Test updates --- .../labkey/api/exp/api/ExperimentService.java | 2 +- .../test/integration/SampleTypeCrud.ispec.ts | 21 +++++++++++++++++-- .../experiment/api/ExpSampleTypeTestCase.jsp | 1 + .../api/SampleTypeUpdateServiceDI.java | 2 -- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/api/src/org/labkey/api/exp/api/ExperimentService.java b/api/src/org/labkey/api/exp/api/ExperimentService.java index b1f3fd629cc..2ce76f42943 100644 --- a/api/src/org/labkey/api/exp/api/ExperimentService.java +++ b/api/src/org/labkey/api/exp/api/ExperimentService.java @@ -147,7 +147,7 @@ static void setInstance(ExperimentService impl) enum QueryOptions { - UseLsidForUpdate, // TODO: Use as marker for behaviors + UseLsidForUpdate, GetSampleRecomputeCol, SkipBulkRemapCache, DeferRequiredLineageValidation, diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index 46c70b1059f..656632f1864 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -1070,13 +1070,17 @@ describe('Amount/Unit CRUD', () => { expect(errorMsg.text).toContain(NEGATIVE_ERROR); errorMsg = await ExperimentCRUDUtils.importSample(server, "Name\tStoredAmount\tUnits\n" + dataName + "\t-1.1\tkg", dataType, "MERGE", topFolderOptions, editorUserOptions); expect(errorMsg.text).toContain(NEGATIVE_ERROR); + + // Using row-by-row await server.post('query', 'updateRows', { schemaName: 'samples', queryName: dataType, rows: [{ Amount: -1, Units: 'kg', - rowId: sampleRowId + rowId: sampleRowId, + },{ + rowId: sampleRowId, }] }, { ...topFolderOptions, ...editorUserOptions }).expect((result) => { const errorResp = JSON.parse(result.text); @@ -1084,11 +1088,24 @@ describe('Amount/Unit CRUD', () => { expect(errorResp['exception']).toContain("Value '-1000.0 (g)' for field 'Amount' is invalid. Amounts must be non-negative."); }); + // Using data iterator + await server.post('query', 'updateRows', { + schemaName: 'samples', + queryName: dataType, + rows: [{ + Amount: -1, + Units: 'kg', + rowId: sampleRowId, + }] + }, { ...topFolderOptions, ...editorUserOptions }).expect((result) => { + const errorResp = JSON.parse(result.text); + expect(errorResp['exception']).toContain("Value '-1' for field 'Amount' is invalid. Amounts must be non-negative."); + }); + errorMsg = await ExperimentCRUDUtils.importCrossTypeData(server, "Name\tStoredAmount\tUnits\tSampleType\nData1\t-1.1\tkg\t" + dataType ,'UPDATE', topFolderOptions, adminOptions, true); expect(errorMsg.text).toContain(NEGATIVE_ERROR); errorMsg = await ExperimentCRUDUtils.importCrossTypeData(server, "Name\tStoredAmount\tUnits\tSampleType\nData1\t-1.1\tkg\t" + dataType ,'MERGE', topFolderOptions, adminOptions, true); expect(errorMsg.text).toContain(NEGATIVE_ERROR); - }); it ("Test units conversion on insert/update", async () => { diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index f8ce588465b..e5e5a328283 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -991,6 +991,7 @@ public void testSampleTypeWithVocabularyProperties() throws Exception oldKey.put("RowId", insertedSample.get(0).get("RowId")); oldKeys.add(oldKey); + // TODO: Either support update of property columns via data iterator or watch out for this case and fallback to _update var updatedSample = helper.updateRows(c, rowsToUpdate, oldKeys, sampleName, schema); assertEquals("Custom Property is not updated", updatedSampleType, OntologyManager.getPropertyObjects(c, updatedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 326d90eef92..f2ff5b87546 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -1654,8 +1654,6 @@ public DataIterator getDataIterator(DataIteratorContext context) } } - if (context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) - drop.remove("lsid"); if (!drop.isEmpty()) source = new DropColumnsDataIterator(source, drop); From ca2be6cb41261bcaafa007349c57393dc30a67f2 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 1 Dec 2025 10:50:41 -0800 Subject: [PATCH 21/62] Vocab properties not supported via iterator --- .../api/query/AbstractQueryUpdateService.java | 2 +- .../labkey/experiment/ExpDataIterators.java | 17 ++++------ .../experiment/api/ExpMaterialTableImpl.java | 4 +-- .../api/SampleTypeUpdateServiceDI.java | 34 +++++++++++++++---- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index bc11979ccbf..075e7fef4e7 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -808,7 +808,7 @@ final protected Map updateOneRow(User user, Container container, // used by updateRows to check if all rows have the same set of keys // prepared statement can only be used to updateRows if all rows have the same set of keys - protected boolean hasUniformKeys(List> rowsToUpdate) + protected static boolean hasUniformKeys(List> rowsToUpdate) { if (rowsToUpdate == null || rowsToUpdate.isEmpty()) return false; diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 7aa150e3ed1..8fb791eb0ee 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2468,7 +2468,10 @@ public DataIterator getDataIterator(DataIteratorContext context) if (validate.hasValidators()) di = validate; - return LoggingDataIterator.wrap(new SampleUpdateNamePolicyDataIterator(di, context, _container)); + if (!NameExpressionOptionService.get().getAllowUserSpecificNamesValue(_container)) + return LoggingDataIterator.wrap(new SampleUpdateNamePolicyDataIterator(di, context)); + + return LoggingDataIterator.wrap(di); } } @@ -2477,15 +2480,11 @@ private static class SampleUpdateNamePolicyDataIterator extends WrapperDataItera private final DataIteratorContext _context; private final Integer _nameCol; - protected SampleUpdateNamePolicyDataIterator(DataIterator di, DataIteratorContext context, Container container) + protected SampleUpdateNamePolicyDataIterator(DataIterator di, DataIteratorContext context) { super(di); _context = context; - - if (NameExpressionOptionService.get().getAllowUserSpecificNamesValue(container)) - _nameCol = null; - else - _nameCol = DataIteratorUtil.createColumnNameMap(di).get(Name.name()); + _nameCol = DataIteratorUtil.createColumnNameMap(di).get(Name.name()); } @Override @@ -2545,14 +2544,10 @@ record TypeData( private final int _dataIdIndex; private final Map> _idsPerType = new HashMap<>(); private final Map> _parentIdsPerType = new HashMap<>(); - private final Map _containerMap = new CaseInsensitiveHashMap<>(); - private final boolean _isCrossFolderUpdate; - private final TSVWriter _tsvWriter; - public MultiDataTypeCrossProjectDataIterator(DataIterator di, DataIteratorContext context, Container container, User user, boolean isCrossType, boolean isCrossFolder, ExpObject dataType, boolean isSamples) { super(di); diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 2facc062a1b..c7541622c44 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -1777,13 +1777,13 @@ public List getUniqueIndices() { // Rewrite the "idx_material_ak" unique index over "Folder", "SampleSet", "Name" to just "Name" // Issue 25397: Don't include the "idx_material_ak" index if the "Name" column hasn't been added to the table. - // Some FKs to ExpMaterialTable don't include the "Name" column (e.g. NabBaseTable.Specimen) + // Some FKs to ExpMaterialTable don't include the "Name" column (e.g., NabBaseTable.Specimen) String indexName = "idx_material_ak"; List ret = new ArrayList<>(super.getUniqueIndices()); if (getColumn("Name") != null) ret.add(new IndexDefinition(indexName, IndexType.Unique, Arrays.asList(getColumn("Name")), null)); else - ret.removeIf( def -> def.name().equals(indexName)); + ret.removeIf(def -> def.name().equals(indexName)); return Collections.unmodifiableList(ret); } diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index f2ff5b87546..b561e8d59ed 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -81,6 +81,7 @@ import org.labkey.api.exp.api.SampleTypeService; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.PropertyService; import org.labkey.api.exp.query.ExpMaterialTable; import org.labkey.api.exp.query.ExpSchema; import org.labkey.api.exp.query.SamplesSchema; @@ -523,6 +524,31 @@ public static void confirmAmountAndUnitsColumns(Collection columns) throw new ConversionExceptionWithMessage(MISSING_UNITS_ERROR_MESSAGE); } + private static boolean useDataIteratorForUpdate( + Container container, + List> rows, + List> oldKeys + ) + { + if (rows == null || rows.isEmpty() || oldKeys != null) + return false; + + Set columnNames = rows.get(0).keySet(); + + // For backwards compatibility we continue to allow specification of an LSID as a key iff RowId is not provided. + // This is only supported via row-by-row update. + boolean useDib = !(columnNames.contains(LSID.name()) && !columnNames.contains(RowId.name())); + + // All rows must have a uniform set of keys for the data iterator to work. + useDib = useDib && hasUniformKeys(rows); + + // Updating vocabulary column values is not supported via data iterator. The underlying StatementUtils expects + // the getObjectURIColumnName() ("LSID" in the case of samples) column to reside on the provisioned table. + useDib = useDib && PropertyService.get().findVocabularyProperties(container, columnNames).isEmpty(); + + return useDib; + } + @Override public List> updateRows( User user, @@ -535,15 +561,9 @@ public List> updateRows( ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException { assert _sampleType != null : "SampleType required for insert/update, but not required for read/delete"; - - // TODO: Perhaps we invert and if an lsid is included we do row-by-row? For backwards compatibility. - // This would required joining in exp.material to get rowId or name. - boolean useDib = false; if (rows != null && !rows.isEmpty()) - { confirmAmountAndUnitsColumns(rows.get(0).keySet()); - useDib = hasUniformKeys(rows); - } + boolean useDib = useDataIteratorForUpdate(container, rows, oldKeys); List> results; DbScope scope = getSchema().getDbSchema().getScope(); From 7b53c5e816047af79de18c20f551a30664c2bbcd Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 1 Dec 2025 12:30:00 -0800 Subject: [PATCH 22/62] Support vocab prop update --- .../labkey/experiment/ExpDataIterators.java | 2 +- .../experiment/api/ExpMaterialTableImpl.java | 186 ++++++------------ .../experiment/api/ExpSampleTypeTestCase.jsp | 1 - .../experiment/api/SampleTypeServiceImpl.java | 7 +- .../api/SampleTypeUpdateServiceDI.java | 7 +- 5 files changed, 65 insertions(+), 138 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 8fb791eb0ee..bf1a59e0414 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2391,6 +2391,7 @@ public DataIterator getDataIterator(DataIteratorContext context) dib = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(dib, _expTable, _container) .setKeyColumns(keyColumns) .setDontUpdate(dontUpdate) + .setVocabularyProperties(PropertyService.get().findVocabularyProperties(_container, colNameMap.keySet())) .setAddlSkipColumns(_excludedColumns) .setCommitRowsBeforeContinuing(true) .setFailOnEmptyUpdate(false)); @@ -2399,7 +2400,6 @@ public DataIterator getDataIterator(DataIteratorContext context) dib = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(dib, _propertiesTable, _container) .setKeyColumns(propertyKeyColumns) .setDontUpdate(dontUpdate) - .setVocabularyProperties(PropertyService.get().findVocabularyProperties(_container, colNameMap.keySet())) .setRemapSchemaColumns(((UpdateableTableInfo) _expTable).remapSchemaColumns()) .setFailOnEmptyUpdate(false)); diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index c7541622c44..e4e8c33c663 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -95,7 +95,6 @@ import org.labkey.api.query.QueryUpdateService; import org.labkey.api.query.QueryUrls; import org.labkey.api.query.RowIdForeignKey; -import org.labkey.api.query.SchemaKey; import org.labkey.api.query.UserSchema; import org.labkey.api.query.column.BuiltInColumnTypes; import org.labkey.api.search.SearchService; @@ -241,7 +240,14 @@ public MutableColumnInfo createColumn(String alias, Column column) } case LSID -> { - return wrapColumn(alias, _rootTable.getColumn(LSID.name())); + var columnInfo = wrapColumn(alias, _rootTable.getColumn(LSID.name())); + columnInfo.setHidden(true); + columnInfo.setReadOnly(true); + columnInfo.setUserEditable(false); + columnInfo.setShownInInsertView(false); + columnInfo.setShownInDetailsView(false); + columnInfo.setShownInUpdateView(false); + return columnInfo; } case MaterialSourceId -> { @@ -272,6 +278,11 @@ public StringExpression getURL(ColumnInfo parent) var columnInfo = wrapColumn(alias, _rootTable.getColumn(RootMaterialRowId.name())); columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), RowId.name())); columnInfo.setLabel("Root Material"); + columnInfo.setHidden(true); + columnInfo.setReadOnly(true); + columnInfo.setShownInInsertView(false); + columnInfo.setShownInDetailsView(false); + columnInfo.setShownInUpdateView(false); columnInfo.setUserEditable(false); // NK: Here we mark the column as not required AND nullable which is the opposite of the database where @@ -288,6 +299,12 @@ public StringExpression getURL(ColumnInfo parent) columnInfo.setSqlTypeName("lsidtype"); columnInfo.setFk(getExpSchema().getMaterialForeignKey(getLookupContainerFilter(), LSID.name())); columnInfo.setLabel("Aliquoted From Parent"); + columnInfo.setHidden(true); + columnInfo.setReadOnly(true); + columnInfo.setUserEditable(false); + columnInfo.setShownInInsertView(false); + columnInfo.setShownInDetailsView(false); + columnInfo.setShownInUpdateView(false); return columnInfo; } case IsAliquot -> @@ -403,23 +420,32 @@ public StringExpression getURL(ColumnInfo parent) case SampleSet -> { var columnInfo = wrapColumn(alias, _rootTable.getColumn("CpasType")); - // NOTE: populateColumns() overwrites this with a QueryForeignKey. Can this be removed? - columnInfo.setFk(new LookupForeignKey(getContainerFilter(), null, null, null, null, "LSID", "Name") + columnInfo.setFk(new QueryForeignKey(_userSchema, getContainerFilter(), ExpSchema.SCHEMA_NAME, getContainer(), null, ExpSchema.TableType.SampleSets.name(), "lsid", null) { @Override - public TableInfo getLookupTableInfo() + protected ContainerFilter getLookupContainerFilter() { - ExpSampleTypeTable sampleTypeTable = ExperimentService.get().createSampleTypeTable(ExpSchema.TableType.SampleSets.toString(), _userSchema, getLookupContainerFilter()); - sampleTypeTable.populate(); - return sampleTypeTable; - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return super.getURL(parent, true); + // Be sure that we can resolve the sample type if it's defined in a separate container. + // Same as CurrentPlusProjectAndShared but includes SampleSet's container as well. + // Issue 37982: Sample Type: Link to precursor sample type does not resolve correctly if sample has + // parents in current sample type and a sample type in the parent container + Set containers = new HashSet<>(); + if (null != getSampleType()) + containers.add(getSampleType().getContainer()); + containers.add(getContainer()); + if (getContainer().getProject() != null) + containers.add(getContainer().getProject()); + containers.add(ContainerManager.getSharedContainer()); + ContainerFilter cf = new ContainerFilter.CurrentPlusExtras(_userSchema.getContainer(), _userSchema.getUser(), containers); + + if (null != _containerFilter && _containerFilter.getType() != ContainerFilter.Type.Current) + cf = new UnionContainerFilter(_containerFilter, cf); + return cf; } }); + columnInfo.setReadOnly(true); + columnInfo.setUserEditable(false); + columnInfo.setShownInInsertView(false); return columnInfo; } case SourceProtocolLSID -> @@ -479,7 +505,10 @@ public StringExpression getURL(ColumnInfo parent) case Run -> { var ret = wrapColumn(alias, _rootTable.getColumn("RunId")); + ret.setFk(getExpSchema().getRunIdForeignKey(getContainerFilter())); ret.setReadOnly(true); + ret.setShownInInsertView(false); + ret.setShownInUpdateView(false); return ret; } case RowId -> @@ -541,7 +570,7 @@ public StringExpression getURL(ColumnInfo parent) } case SampleState -> { - boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); + boolean statusEnabled = isStatusEnabled(getContainer()); var ret = wrapColumn(alias, _rootTable.getColumn(column.name())); ret.setLabel("Status"); ret.setHidden(!statusEnabled); @@ -648,7 +677,7 @@ public StringExpression getURL(ColumnInfo parent) } case MaterialExpDate -> { - var ret = wrapColumn(alias, _rootTable.getColumn("MaterialExpDate")); + var ret = wrapColumn(alias, _rootTable.getColumn(MaterialExpDate.name())); ret.setLabel("Expiration Date"); ret.setShownInDetailsView(true); ret.setShownInInsertView(true); @@ -659,35 +688,9 @@ public StringExpression getURL(ColumnInfo parent) } } - @Override - public MutableColumnInfo createPropertyColumn(String alias) + private static boolean isStatusEnabled(Container c) { - var ret = super.createPropertyColumn(alias); - if (_ss != null) - { - final TableInfo t = _ss.getTinfo(); - if (t != null) - { - ret.setFk(new LookupForeignKey() - { - @Override - public TableInfo getLookupTableInfo() - { - return t; - } - - @Override - protected ColumnInfo getPkColumn(TableInfo table) - { - return t.getColumn("lsid"); // TODO: Seems to be pointing at the wrong table - } - }); - } - } - ret.setIsUnselectable(true); - ret.setDescription("A holder for any custom fields associated with this sample"); - ret.setHidden(true); - return ret; + return SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(c).isEmpty(); } private Unit getSampleTypeUnit() @@ -745,7 +748,9 @@ protected void populateColumns() addColumn(RunApplicationOutput); addColumn(SourceProtocolLSID); + List defaultCols = new ArrayList<>(); var nameCol = addColumn(Name); + defaultCols.add(Name.fieldKey()); if (st != null && st.hasNameAsIdCol()) { // Show the Name field but don't mark is as required when using name expressions @@ -771,97 +776,28 @@ protected void populateColumns() addColumn(Alias); addColumn(Description); - - var typeColumnInfo = addColumn(SampleSet); - typeColumnInfo.setFk(new QueryForeignKey(_userSchema, getContainerFilter(), ExpSchema.SCHEMA_NAME, getContainer(), null, ExpSchema.TableType.SampleSets.name(), "lsid", null) - { - @Override - protected ContainerFilter getLookupContainerFilter() - { - // Be sure that we can resolve the sample type if it's defined in a separate container. - // Same as CurrentPlusProjectAndShared but includes SampleSet's container as well. - // Issue 37982: Sample Type: Link to precursor sample type does not resolve correctly if sample has - // parents in current sample type and a sample type in the parent container - Set containers = new HashSet<>(); - if (null != st) - containers.add(st.getContainer()); - containers.add(getContainer()); - if (getContainer().getProject() != null) - containers.add(getContainer().getProject()); - containers.add(ContainerManager.getSharedContainer()); - ContainerFilter cf = new ContainerFilter.CurrentPlusExtras(_userSchema.getContainer(), _userSchema.getUser(), containers); - - if (null != _containerFilter && _containerFilter.getType() != ContainerFilter.Type.Current) - cf = new UnionContainerFilter(_containerFilter, cf); - return cf; - } - }); - - typeColumnInfo.setReadOnly(true); - typeColumnInfo.setUserEditable(false); - typeColumnInfo.setShownInInsertView(false); - + addColumn(SampleSet); addColumn(MaterialExpDate); + defaultCols.add(MaterialExpDate.fieldKey()); addContainerColumn(Folder, null); - var runCol = addColumn(Run); - runCol.setFk(new ExpSchema(_userSchema.getUser(), getContainer()).getRunIdForeignKey(getContainerFilter())); - runCol.setShownInInsertView(false); - runCol.setShownInUpdateView(false); - - var colLSID = addColumn(LSID); - colLSID.setHidden(true); - colLSID.setReadOnly(true); - colLSID.setUserEditable(false); - colLSID.setShownInInsertView(false); - colLSID.setShownInDetailsView(false); - colLSID.setShownInUpdateView(false); - - var rootRowId = addColumn(RootMaterialRowId); - rootRowId.setHidden(true); - rootRowId.setReadOnly(true); - rootRowId.setUserEditable(false); - rootRowId.setShownInInsertView(false); - rootRowId.setShownInDetailsView(false); - rootRowId.setShownInUpdateView(false); - - var aliquotParentLSID = addColumn(AliquotedFromLSID); - aliquotParentLSID.setHidden(true); - aliquotParentLSID.setReadOnly(true); - aliquotParentLSID.setUserEditable(false); - aliquotParentLSID.setShownInInsertView(false); - aliquotParentLSID.setShownInDetailsView(false); - aliquotParentLSID.setShownInUpdateView(false); - + if (getContainer().hasProductFolders()) + defaultCols.add(Folder.fieldKey()); + addColumn(Run); + defaultCols.add(Run.fieldKey()); + addColumn(LSID); + addColumn(RootMaterialRowId); + addColumn(AliquotedFromLSID); addColumn(IsAliquot); addColumn(Created); addColumn(CreatedBy); addColumn(Modified); addColumn(ModifiedBy); - - List defaultCols = new ArrayList<>(); - defaultCols.add(Name.fieldKey()); - defaultCols.add(MaterialExpDate.fieldKey()); - boolean hasProductFolders = getContainer().hasProductFolders(); - if (hasProductFolders) - defaultCols.add(Folder.fieldKey()); - defaultCols.add(Run.fieldKey()); - if (st == null) defaultCols.add(SampleSet.fieldKey()); - addColumn(Flag); - - var statusColInfo = addColumn(SampleState); - boolean statusEnabled = SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(getContainer()).isEmpty(); - statusColInfo.setShownInDetailsView(statusEnabled); - statusColInfo.setShownInInsertView(statusEnabled); - statusColInfo.setShownInUpdateView(statusEnabled); - statusColInfo.setHidden(!statusEnabled); - statusColInfo.setRemapMissingBehavior(SimpleTranslator.RemapMissingBehavior.Error); - if (statusEnabled) + addColumn(SampleState); + if (isStatusEnabled(getContainer())) defaultCols.add(SampleState.fieldKey()); - statusColInfo.setFk(new QueryForeignKey.Builder(getUserSchema(), getSampleStatusLookupContainerFilter()) - .schema(getExpSchema()).table(ExpSchema.TableType.SampleStatus).display("Label")); // TODO is this a real Domain??? if (st != null && !"urn:lsid:labkey.com:SampleSource:Default".equals(st.getDomain().getTypeURI())) @@ -1190,7 +1126,7 @@ private void addSampleTypeColumns(ExpSampleType st, List visibleColumn if (null != dp) { PropertyColumn.copyAttributes(schema.getUser(), propColumn, dp.getPropertyDescriptor(), schema.getContainer(), - SchemaKey.fromParts("samples"), st.getName(), RowId.fieldKey(), null, getLookupContainerFilter()); + SamplesSchema.SCHEMA_SAMPLES, st.getName(), RowId.fieldKey(), null, getLookupContainerFilter()); if (idCols.contains(dp)) { diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index e5e5a328283..f8ce588465b 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -991,7 +991,6 @@ public void testSampleTypeWithVocabularyProperties() throws Exception oldKey.put("RowId", insertedSample.get(0).get("RowId")); oldKeys.add(oldKey); - // TODO: Either support update of property columns via data iterator or watch out for this case and fallback to _update var updatedSample = helper.updateRows(c, rowsToUpdate, oldKeys, sampleName, schema); assertEquals("Custom Property is not updated", updatedSampleType, OntologyManager.getPropertyObjects(c, updatedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 17227bfb785..882e1ec3559 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -698,11 +698,8 @@ public void deleteSampleType(long rowId, Container c, User user, @Nullable Strin // Delete sequences (genId and the unique counters) DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); - SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); - QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); - - SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); - QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); + QueryService.get().fireQueryDeleted(user, c, null, SamplesSchema.SCHEMA_SAMPLES, singleton(source.getName())); + QueryService.get().fireQueryDeleted(user, c, null, ExpSchema.SCHEMA_EXP_MATERIALS, singleton(source.getName())); // Remove SampleType from search index try (Timing ignored = MiniProfiler.step("search docs")) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index b561e8d59ed..f61e46f7322 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -525,7 +525,6 @@ public static void confirmAmountAndUnitsColumns(Collection columns) } private static boolean useDataIteratorForUpdate( - Container container, List> rows, List> oldKeys ) @@ -542,10 +541,6 @@ private static boolean useDataIteratorForUpdate( // All rows must have a uniform set of keys for the data iterator to work. useDib = useDib && hasUniformKeys(rows); - // Updating vocabulary column values is not supported via data iterator. The underlying StatementUtils expects - // the getObjectURIColumnName() ("LSID" in the case of samples) column to reside on the provisioned table. - useDib = useDib && PropertyService.get().findVocabularyProperties(container, columnNames).isEmpty(); - return useDib; } @@ -563,7 +558,7 @@ public List> updateRows( assert _sampleType != null : "SampleType required for insert/update, but not required for read/delete"; if (rows != null && !rows.isEmpty()) confirmAmountAndUnitsColumns(rows.get(0).keySet()); - boolean useDib = useDataIteratorForUpdate(container, rows, oldKeys); + boolean useDib = useDataIteratorForUpdate(rows, oldKeys); List> results; DbScope scope = getSchema().getDbSchema().getScope(); From b35df9a247185afee0f7bf3c7402860072e30980 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 1 Dec 2025 14:48:44 -0800 Subject: [PATCH 23/62] comments --- .../org/labkey/experiment/api/SampleTypeUpdateServiceDI.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index f61e46f7322..2cd59ead2aa 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -886,7 +886,6 @@ protected Map updateRow(User user, Container container, Map _update(User user, Container c, Map _update(User user, Container c, Map> getExistingRows( Map rowNumLsid = new IntHashMap<>(); Map rowIdRowNumMap = new LinkedHashMap<>(); - // TODO: What if we didn't support lsidRowMap? Map lsidRowNumMap = new CaseInsensitiveMapWrapper<>(new LinkedHashMap<>()); Map nameRowNumMap = new LinkedHashMap<>(); Integer sampleTypeId = null; From 04e9bb13658e1477863b7345f53aa40c92ca464f Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 1 Dec 2025 15:58:55 -0800 Subject: [PATCH 24/62] Upgrade code --- .../experiment/ExperimentUpgradeCode.java | 158 ++++++++++++++++++ .../api/property/StorageProvisionerImpl.java | 2 +- 2 files changed, 159 insertions(+), 1 deletion(-) diff --git a/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java b/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java index e1b97b83aca..4a8794328bd 100644 --- a/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java +++ b/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java @@ -17,31 +17,46 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.AuditLogService; import org.labkey.api.audit.AuditTypeEvent; import org.labkey.api.audit.SampleTimelineAuditEvent; import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; import org.labkey.api.data.JdbcType; import org.labkey.api.data.Parameter; import org.labkey.api.data.ParameterMapStatement; import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyStorageSpec; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfo; import org.labkey.api.data.Selector; import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; import org.labkey.api.data.UpgradeCode; +import org.labkey.api.exp.PropertyDescriptor; import org.labkey.api.exp.api.ExpSampleType; import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.SampleTypeDomainKind; import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleLoader; import org.labkey.api.ontology.Unit; import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryService; import org.labkey.api.security.LimitedUser; import org.labkey.api.security.User; @@ -49,12 +64,19 @@ import org.labkey.api.settings.AppProps; import org.labkey.api.util.logging.LogHelper; import org.labkey.experiment.api.ClosureQueryHelper; +import org.labkey.experiment.api.ExpSampleTypeImpl; +import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.experiment.api.MaterialSource; +import org.labkey.experiment.api.property.DomainImpl; +import org.labkey.experiment.api.property.DomainPropertyImpl; +import org.labkey.experiment.api.property.StorageProvisionerImpl; import org.labkey.experiment.samples.SampleTimelineAuditProvider; import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -406,6 +428,142 @@ private static int convertAmountsToBaseUnits(Container container, User user) { throw new RuntimeException(e); } + } + + /** + * Called from exp-25.013-25.014.sql + */ + @SuppressWarnings("unused") + public static void dropProvisionedSampleTypeLsidColumn(ModuleContext context) + { + if (context.isNewInstall()) + return; + + try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) + { + // Process all sample types across all containers + TableInfo sampleTypeTable = ExperimentServiceImpl.get().getTinfoSampleType(); + List sampleTypes = new TableSelector(sampleTypeTable, null, null) + .stream(MaterialSource.class) + .map(ExpSampleTypeImpl::new) + .toList(); + + LOG.info("Dropping the lsid column from {} sample types", sampleTypes.size()); + + int successCount = 0; + for (ExpSampleTypeImpl st : sampleTypes) + { + boolean success = dropSampleLsid(st); + if (success) + successCount++; + } + + LOG.info("Dropped lsid column from {} of {} sample types successfully.", successCount, sampleTypes.size()); + + if (sampleTypeTable != null) + throw new IllegalArgumentException("We must stop this work!"); + + tx.commit(); + } + } + + private static boolean dropSampleLsid(ExpSampleTypeImpl st) + { + ProvisionedSampleTypeContext context = getProvisionedSampleTypeContext(st); + if (context == null) + return false; + + Domain domain = context.domain; + TableInfo table = context.provisionedTable; + + String lsidColumnName = "lsid"; + ColumnInfo lsidColumn = table.getColumn(FieldKey.fromParts(lsidColumnName)); + if (lsidColumn == null) + { + LOG.info("No lsid column found on table '{}'. Skipping drop.", table.getName()); + return false; + } + + Set indicesToRemove = new HashSet<>(); + for (var index : table.getAllIndices()) + { + var indexColumns = index.columns(); + if (indexColumns.contains(lsidColumn)) + { + // We only expect to be dropping indices on the LSID column alone. However, if we encounter + // another index on the provisioned table, log information about the index and continue to remove it. + if (indexColumns.size() > 1) + LOG.info("Dropping index '{}' on table '{}' because it contains the lsid column.", index.name(), table.getName()); + + indicesToRemove.add(index.name()); + } + } + + if (!indicesToRemove.isEmpty()) + StorageProvisionerImpl.get().dropTableIndices(domain, indicesToRemove); + else + LOG.info("No indices found on table '{}' that contain the lsid column.", table.getName()); + + // Remanufacture a property descriptor that matches the original LSID property descriptor. + var spec = new PropertyStorageSpec(lsidColumnName, JdbcType.VARCHAR, 300).setNullable(false); + PropertyDescriptor pd = new PropertyDescriptor(); + pd.setContainer(st.getContainer()); + pd.setDatabaseDefaultValue(spec.getDefaultValue()); + pd.setName(spec.getName()); + pd.setJdbcType(spec.getJdbcType(), spec.getSize()); + pd.setNullable(spec.isNullable()); + pd.setMvEnabled(spec.isMvEnabled()); + pd.setPropertyURI(DomainUtil.createUniquePropertyURI(domain.getTypeURI(), null, new CaseInsensitiveHashSet())); + pd.setDescription(spec.getDescription()); + pd.setImportAliases(spec.getImportAliases()); + pd.setScale(spec.getSize()); + DomainPropertyImpl dp = new DomainPropertyImpl((DomainImpl) domain, pd); + + LOG.debug("Dropping lsid column from table '{}' for sample type '{}' in folder {}.", table.getName(), st.getName(), st.getContainer().getPath()); + StorageProvisionerImpl.get().dropProperties(domain, Set.of(dp)); + + return true; + } + + private record ProvisionedSampleTypeContext(Domain domain, SchemaTableInfo provisionedTable) {} + + private static @Nullable ProvisionedSampleTypeContext getProvisionedSampleTypeContext(@NotNull ExpSampleTypeImpl st) + { + Domain domain = st.getDomain(); + SampleTypeDomainKind kind = null; + try + { + kind = (SampleTypeDomainKind) domain.getDomainKind(); + } + catch (IllegalArgumentException e) + { + // pass + } + + if (kind == null) + { + LOG.info("Sample type '" + st.getName() + "' (" + st.getRowId() + ") has no domain kind."); + return null; + } + else if (kind.getStorageSchemaName() == null) + { + // e.g., SpecimenSampleTypeDomainKind is not provisioned + LOG.info("Sample type '" + st.getName() + "' (" + st.getRowId() + ") has no provisioned storage schema."); + return null; + } + + DbSchema schema = kind.getSchema(); + StorageProvisioner.get().ensureStorageTable(domain, kind, schema.getScope()); + domain = PropertyService.get().getDomain(domain.getTypeId()); + assert (null != domain && null != domain.getStorageTableName()); + + SchemaTableInfo provisionedTable = schema.getTable(domain.getStorageTableName()); + if (provisionedTable == null) + { + LOG.error("Sample type '" + st.getName() + "' (" + st.getRowId() + ") has no provisioned table."); + return null; + } + return new ProvisionedSampleTypeContext(domain, provisionedTable); } } diff --git a/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java b/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java index 6ebc330eea3..727423a1a70 100644 --- a/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java +++ b/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java @@ -892,7 +892,7 @@ private DbScope validateDomain(Domain domain) return scope; } - private void dropTableIndices(Domain domain, Set indexNames) + public void dropTableIndices(Domain domain, Set indexNames) { DbScope scope = validateDomain(domain); From d3be632d23368fdecc9b7a46800360607660e330 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 1 Dec 2025 16:02:18 -0800 Subject: [PATCH 25/62] scripts --- .../schemas/dbscripts/postgresql/exp-25.014-25.015.sql | 1 + .../resources/schemas/dbscripts/sqlserver/exp-25.014-25.015.sql | 1 + experiment/src/org/labkey/experiment/ExperimentModule.java | 2 +- experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 experiment/resources/schemas/dbscripts/postgresql/exp-25.014-25.015.sql create mode 100644 experiment/resources/schemas/dbscripts/sqlserver/exp-25.014-25.015.sql diff --git a/experiment/resources/schemas/dbscripts/postgresql/exp-25.014-25.015.sql b/experiment/resources/schemas/dbscripts/postgresql/exp-25.014-25.015.sql new file mode 100644 index 00000000000..249f1a480a6 --- /dev/null +++ b/experiment/resources/schemas/dbscripts/postgresql/exp-25.014-25.015.sql @@ -0,0 +1 @@ +SELECT core.executeJavaUpgradeCode('dropProvisionedSampleTypeLsidColumn'); diff --git a/experiment/resources/schemas/dbscripts/sqlserver/exp-25.014-25.015.sql b/experiment/resources/schemas/dbscripts/sqlserver/exp-25.014-25.015.sql new file mode 100644 index 00000000000..01ff3f93ca7 --- /dev/null +++ b/experiment/resources/schemas/dbscripts/sqlserver/exp-25.014-25.015.sql @@ -0,0 +1 @@ +EXEC core.executeJavaUpgradeCode 'dropProvisionedSampleTypeLsidColumn'; diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 0b4da589654..7ccbc8684d1 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -200,7 +200,7 @@ public String getName() @Override public Double getSchemaVersion() { - return 25.014; + return 25.015; } @Nullable diff --git a/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java b/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java index 4a8794328bd..0f564ee6b6a 100644 --- a/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java +++ b/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java @@ -431,7 +431,7 @@ private static int convertAmountsToBaseUnits(Container container, User user) } /** - * Called from exp-25.013-25.014.sql + * Called from exp-25.014-25.015.sql */ @SuppressWarnings("unused") public static void dropProvisionedSampleTypeLsidColumn(ModuleContext context) From ee42ee99d3d6806808a2436d36559ab700df82d1 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 1 Dec 2025 16:05:49 -0800 Subject: [PATCH 26/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index e1ab00f132b..d1503713511 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "6.72.2-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.2-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-qZOOfQnFML7rkmdOHQrX1H0if3hpqZ0Yj2rEXAtzfpFgx06XUEZggF9UUKtURoxIccPjOhuy4m64BLEbT/DCaw==", + "version": "7.0.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.0.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-KCgJc3ZQv8kUUfNI0pUfJYLwUgt6s/CyhRpnW0s3sPbAl9hQFrEZVbwrADDCXWMXGFhujY2A+AL8a0OFHfymKw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index 58ea6ada746..32bba39725c 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index bc4b408216e..e468b5d6e11 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "6.72.2-fb-remove-sample-lsid.0", + "@labkey/components": "7.0.1-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "6.72.2-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.2-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-qZOOfQnFML7rkmdOHQrX1H0if3hpqZ0Yj2rEXAtzfpFgx06XUEZggF9UUKtURoxIccPjOhuy4m64BLEbT/DCaw==", + "version": "7.0.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.0.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-KCgJc3ZQv8kUUfNI0pUfJYLwUgt6s/CyhRpnW0s3sPbAl9hQFrEZVbwrADDCXWMXGFhujY2A+AL8a0OFHfymKw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index a77037c1c10..15cf987d66d 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "6.72.2-fb-remove-sample-lsid.0", + "@labkey/components": "7.0.1-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index f328ceb2ab1..f2c7767049c 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "6.72.2-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.2-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-qZOOfQnFML7rkmdOHQrX1H0if3hpqZ0Yj2rEXAtzfpFgx06XUEZggF9UUKtURoxIccPjOhuy4m64BLEbT/DCaw==", + "version": "7.0.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.0.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-KCgJc3ZQv8kUUfNI0pUfJYLwUgt6s/CyhRpnW0s3sPbAl9hQFrEZVbwrADDCXWMXGFhujY2A+AL8a0OFHfymKw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index f93eb678aad..5e3326f2cd4 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index c3d92aa2aa1..762787abef2 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "6.72.2-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-6.72.2-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-qZOOfQnFML7rkmdOHQrX1H0if3hpqZ0Yj2rEXAtzfpFgx06XUEZggF9UUKtURoxIccPjOhuy4m64BLEbT/DCaw==", + "version": "7.0.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.0.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-KCgJc3ZQv8kUUfNI0pUfJYLwUgt6s/CyhRpnW0s3sPbAl9hQFrEZVbwrADDCXWMXGFhujY2A+AL8a0OFHfymKw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index 99684a98afb..e79ead3b18c 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "6.72.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", From ad59b891353258529e4c8ec3c30d7d0852181fd5 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 1 Dec 2025 16:08:44 -0800 Subject: [PATCH 27/62] Let it happen --- .../src/org/labkey/experiment/ExperimentUpgradeCode.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java b/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java index 0f564ee6b6a..0612cc27174 100644 --- a/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java +++ b/experiment/src/org/labkey/experiment/ExperimentUpgradeCode.java @@ -460,9 +460,6 @@ public static void dropProvisionedSampleTypeLsidColumn(ModuleContext context) LOG.info("Dropped lsid column from {} of {} sample types successfully.", successCount, sampleTypes.size()); - if (sampleTypeTable != null) - throw new IllegalArgumentException("We must stop this work!"); - tx.commit(); } } From c391af5afe72cd9f561251305d03d2b63237e577 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 2 Dec 2025 06:27:32 -0800 Subject: [PATCH 28/62] Rename checks --- .../api/query/AbstractQueryUpdateService.java | 25 +++++++++----- .../api/query/DefaultQueryUpdateService.java | 17 ---------- .../api/SampleTypeUpdateServiceDI.java | 33 +++++++++++-------- 3 files changed, 35 insertions(+), 40 deletions(-) diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index 075e7fef4e7..b90de00d09b 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -269,17 +269,24 @@ protected final DataIteratorContext getDataIteratorContext(BatchValidationExcept context.setInsertOption(forImport); context.setConfigParameters(configParameters); configureDataIteratorContext(context); - if (configParameters != null) + recordDataIteratorUsed(configParameters); + + return context; + } + + protected void recordDataIteratorUsed(@Nullable Map configParameters) + { + if (configParameters == null) + return; + + try { - try - { - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); - } catch (UnsupportedOperationException ignore) - { - // configParameters is immutable, likely originated from a junit test - } + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + } + catch (UnsupportedOperationException ignore) + { + // configParameters is immutable, likely originated from a junit test } - return context; } /** diff --git a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java index adb096bcb33..5538e77a783 100644 --- a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java +++ b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java @@ -21,7 +21,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.ArrayListMap; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveMapWrapper; @@ -71,7 +70,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -935,19 +933,4 @@ protected void configureCrossFolderImport(DataIteratorBuilder rows, DataIterator context.setCrossFolderImport(false); } } - - protected void recordDataIteratorUsed(@Nullable Map configParameters) - { - if (configParameters != null) - { - try - { - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); - } catch (UnsupportedOperationException ignore) - { - // configParameters is immutable, likely originated from a junit test - } - } - } - } diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 2cd59ead2aa..6281f5e089c 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -1361,26 +1361,31 @@ public Map> getExistingRows( for (Map.Entry> keyMap : keys.entrySet()) { Integer rowNum = keyMap.getKey(); - Long rowId = getMaterialRowId(keyMap.getValue()); - String lsid = getMaterialLsid(keyMap.getValue()); - String name = getMaterialName(keyMap.getValue()); - Integer materialSourceId = getMaterialSourceId(keyMap.getValue()); - if (rowId != null) + { rowIdRowNumMap.put(rowId, rowNum); - else if (lsid != null) + continue; + } + + String lsid = getMaterialLsid(keyMap.getValue()); + if (lsid != null) { lsidRowNumMap.put(lsid, rowNum); rowNumLsid.put(rowNum, lsid); + continue; } - else if (name != null && materialSourceId != null) + + String name = getMaterialName(keyMap.getValue()); + Integer materialSourceId = getMaterialSourceId(keyMap.getValue()); + if (name != null && materialSourceId != null) { sampleTypeId = materialSourceId; nameRowNumMap.put(name, rowNum); + continue; } - else - throw new QueryUpdateServiceException("Either RowId or Name is required to get Sample Type Material."); + + throw new QueryUpdateServiceException("Either RowId or Name is required to get Sample Type Material."); } if (!rowIdRowNumMap.isEmpty()) @@ -1390,9 +1395,9 @@ else if (name != null && materialSourceId != null) Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); for (Map row : rows) { - Long rowId = asLong(row.get("rowid")); + Long rowId = asLong(row.get(RowId.name())); Integer rowNum = rowIdRowNumMap.get(rowId); - String sampleLsid = (String) row.get("lsid"); + String sampleLsid = (String) row.get(LSID.name()); rowNumLsid.put(rowNum, sampleLsid); sampleRows.put(rowNum, row); @@ -1412,7 +1417,7 @@ else if (name != null && materialSourceId != null) Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); for (Map row : rows) { - String sampleLsid = (String) row.get("lsid"); + String sampleLsid = (String) row.get(LSID.name()); Integer rowNum = lsidRowNumMap.get(sampleLsid); sampleRows.put(rowNum, row); @@ -1429,9 +1434,9 @@ else if (name != null && materialSourceId != null) Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); for (Map row : rows) { - String name = (String) row.get("name"); + String name = (String) row.get(Name.name()); Integer rowNum = nameRowNumMap.get(name); - String sampleLsid = (String) row.get("lsid"); + String sampleLsid = (String) row.get(LSID.name()); sampleRows.put(rowNum, row); rowNumLsid.put(rowNum, sampleLsid); From 082ecb5cd8ef65846704401e658881d7bb63215e Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 2 Dec 2025 10:47:59 -0800 Subject: [PATCH 29/62] Support updating alias --- .../org/labkey/experiment/ExpDataIterators.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index bf1a59e0414..6f5de172914 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -519,6 +519,7 @@ private static class AliasDataIterator extends ExpDataTypeDataIterator final Supplier _nameCol; Map _lsidAliasMap = new HashMap<>(); private final TableInfo _expAliasTable; + private final boolean _isUpdateOnly; protected AliasDataIterator(DataIterator di, DataIteratorContext context, Container container, User user, TableInfo expAliasTable, ExpObject dataType, boolean isSample) { @@ -526,9 +527,10 @@ protected AliasDataIterator(DataIterator di, DataIteratorContext context, Contai Map map = DataIteratorUtil.createColumnNameMap(di); _aliasCol = map.get(ALIASCOLUMNALIAS) == null ? null : di.getSupplier(map.get(ALIASCOLUMNALIAS)); - _lsidCol = map.get("lsid") == null ? null : di.getSupplier(map.get("lsid")); - _nameCol = map.get("name") == null ? null : di.getSupplier(map.get("name")); + _lsidCol = map.get(LSID.name()) == null ? null : di.getSupplier(map.get(LSID.name())); + _nameCol = map.get(Name.name()) == null ? null : di.getSupplier(map.get(Name.name())); _expAliasTable = expAliasTable; + _isUpdateOnly = _context.getInsertOption().updateOnly; } @Override @@ -549,7 +551,7 @@ public boolean next() throws BatchValidationException // Collect alias values and map them by LSID String lsid = null; - if (_nameCol != null && (_context.getInsertOption().mergeRows || _context.getInsertOption().updateOnly)) + if (_nameCol != null && (_context.getInsertOption().mergeRows || _isUpdateOnly)) { Object nameValue = _nameCol.get(); if (nameValue instanceof String name) @@ -567,6 +569,13 @@ public boolean next() throws BatchValidationException lsid = lsidString; } + if (lsid == null && _isUpdateOnly) + { + Map oldRow = getExistingRecord(); + if (oldRow != null) + lsid = (String) oldRow.get(LSID.name()); + } + if (!StringUtils.isEmpty(lsid)) _lsidAliasMap.put(lsid, _aliasCol.get()); From b7a32613266ba3a7d43d02c51d9f5ddf24c9e1aa Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 2 Dec 2025 13:20:05 -0800 Subject: [PATCH 30/62] Match rename support --- .../labkey/experiment/ExpDataIterators.java | 46 +++++++++++++------ .../api/SampleTypeUpdateServiceDI.java | 10 +++- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 6f5de172914..1020808353e 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -455,11 +455,11 @@ public Object get(int i) Map existingMap = getExistingRecord(); if (existingMap != null && !existingMap.isEmpty()) { - Pair needRecac = determineRecalcFromExistingRecord(i, existingMap); - if (needRecac == null) + Pair needRecalc = determineRecalcFromExistingRecord(i, existingMap); + if (needRecalc == null) return null; - if (needRecac.first && needRecac.second != null) - return needRecac.second; + if (needRecalc.first && needRecalc.second != null) + return needRecalc.second; } // without existing record, or if existing record is missing root information, we have to be conservative and assume this is a new aliquot, or an amount/status update @@ -2477,23 +2477,29 @@ public DataIterator getDataIterator(DataIteratorContext context) if (validate.hasValidators()) di = validate; - if (!NameExpressionOptionService.get().getAllowUserSpecificNamesValue(_container)) - return LoggingDataIterator.wrap(new SampleUpdateNamePolicyDataIterator(di, context)); - - return LoggingDataIterator.wrap(di); + return LoggingDataIterator.wrap(new SampleNameChangeDataIterator(di, context, _container, _user)); } } - private static class SampleUpdateNamePolicyDataIterator extends WrapperDataIterator + private static class SampleNameChangeDataIterator extends WrapperDataIterator { private final DataIteratorContext _context; private final Integer _nameCol; + private final boolean _isAllowUserSpecificNamesValue; + private final User _user; - protected SampleUpdateNamePolicyDataIterator(DataIterator di, DataIteratorContext context) + protected SampleNameChangeDataIterator( + DataIterator di, + DataIteratorContext context, + Container container, + User user + ) { super(di); _context = context; _nameCol = DataIteratorUtil.createColumnNameMap(di).get(Name.name()); + _isAllowUserSpecificNamesValue = NameExpressionOptionService.get().getAllowUserSpecificNamesValue(container); + _user = user; } @Override @@ -2503,15 +2509,27 @@ public boolean next() throws BatchValidationException if (!hasNext) return false; - if (_nameCol == null || _context.getErrors().hasErrors() || getExistingRecord() == null) + var existingRecord = getExistingRecord(); + if (_nameCol == null || _context.getErrors().hasErrors() || existingRecord == null) return true; Object newNameObj = get(_nameCol); String newName = newNameObj == null ? null : String.valueOf(newNameObj); - String oldName = (String) getExistingRecord().get(Name.name()); + String oldName = (String) existingRecord.get(Name.name()); + boolean hasNameChange = !StringUtils.isEmpty(newName) && !newName.equals(oldName); - if (!StringUtils.isEmpty(newName) && !newName.equals(oldName)) - _context.getErrors().addRowError(new ValidationException("User-specified sample name not allowed")); + if (hasNameChange) + { + if (_isAllowUserSpecificNamesValue) + { + Long rowId = asLong(existingRecord.get(RowId.name())); + var sample = ExperimentService.get().getExpMaterial(rowId); + if (sample != null) + ExperimentService.get().addObjectLegacyName(sample.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpMaterial.class), oldName, _user); + } + else + _context.getErrors().addRowError(new ValidationException("User-specified sample name not allowed")); + } return true; } diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 6281f5e089c..6680a1e7642 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -901,12 +901,12 @@ protected Map _update(User user, Container c, Map _update(User user, Container c, Map _update(User user, Container c, Map Date: Wed, 3 Dec 2025 15:31:05 -0800 Subject: [PATCH 31/62] Support external LSID supplier --- .../api/ExpDataClassDataTableImpl.java | 13 ++++--- .../experiment/api/ExpMaterialTableImpl.java | 37 +++++++++---------- .../api/SampleTypeUpdateServiceDI.java | 16 ++++++-- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index 40499f8d336..cc96470d824 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -902,22 +902,23 @@ public Set getAltKeysForUpdate() if (context.getInsertOption().allowUpdate) { - boolean isUpdateUsingLsid = context.getInsertOption().updateOnly && colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); + boolean isUpdateUsingLsid = context.getInsertOption().updateOnly && + colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && + context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); + if (isUpdateUsingLsid) keyColumnNames.add(Column.LSID.name()); else { Set altMergeKeys = getAltMergeKeys(context); - if (altMergeKeys == null) - return null; - - keyColumnNames.addAll(altMergeKeys); + if (altMergeKeys != null) + keyColumnNames.addAll(altMergeKeys); } } else keyColumnNames.add(Column.LSID.name()); - return keyColumnNames; + return keyColumnNames.isEmpty() ? null : keyColumnNames; } @Override diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index e4e8c33c663..a23e9ffc98d 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -1768,33 +1768,32 @@ public Set getAltMergeKeys(DataIteratorContext context) @Override public @Nullable Set getExistingRecordKeyColumnNames(DataIteratorContext context, Map colNameMap) { - if (!context.getInsertOption().allowUpdate) - return null; - Set keyColumnNames = new CaseInsensitiveHashSet(); - if (context.getInsertOption().updateOnly) + + if (context.getInsertOption().allowUpdate) { - if (colNameMap.containsKey(RowId.name())) - keyColumnNames.add(RowId.name()); - else + if (context.getInsertOption().updateOnly) { - for (String altKey : getAltKeysForUpdate()) + if (colNameMap.containsKey(RowId.name())) + keyColumnNames.add(RowId.name()); + else { - if (colNameMap.containsKey(altKey)) - keyColumnNames.add(altKey); + for (String altKey : getAltKeysForUpdate()) + { + if (colNameMap.containsKey(altKey)) + keyColumnNames.add(altKey); + } } } - } - else - { - Set altMergeKeys = getAltMergeKeys(context); - if (altMergeKeys == null) - return null; - - keyColumnNames.addAll(altMergeKeys); + else + { + Set altMergeKeys = getAltMergeKeys(context); + if (altMergeKeys != null) + keyColumnNames.addAll(altMergeKeys); + } } - return keyColumnNames; + return keyColumnNames.isEmpty() ? null : keyColumnNames; } @Override diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 6680a1e7642..38720da5844 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -1678,6 +1678,8 @@ public DataIterator getDataIterator(DataIteratorContext context) } } + if (context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) + drop.remove(LSID.name()); if (!drop.isEmpty()) source = new DropColumnsDataIterator(source, drop); @@ -1815,7 +1817,6 @@ static class _GenerateNamesDataIterator extends SimpleTranslator final List>> _extraPropsFns; final SampleNameGeneratorState _nameState; final Lsid.LsidBuilder lsidBuilder; - final DbSequence _lsidDbSeq; final ExpSampleTypeImpl _sampleType; final User _user; @@ -1858,12 +1859,19 @@ static class _GenerateNamesDataIterator extends SimpleTranslator boolean skipDuplicateCheck = context.getConfigParameterBoolean(SkipMaxSampleCounterFunction); _nameState = sampleType.getNameGenState(skipDuplicateCheck, true, _container, user); lsidBuilder = sampleType.generateSampleLSID(); - _lsidDbSeq = sampleType.getSampleLsidDbSeq(batchSize, sampleType.getContainer()); - selectAll(CaseInsensitiveHashSet.of(Name.name(), LSID.name(), RootMaterialRowId.name())); + boolean useLsidForUpdate = context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); + if (useLsidForUpdate) + selectAll(CaseInsensitiveHashSet.of(Name.name(), RootMaterialRowId.name())); + else + selectAll(CaseInsensitiveHashSet.of(Name.name(), LSID.name(), RootMaterialRowId.name())); addColumn(new BaseColumnInfo("name", JdbcType.VARCHAR), (Supplier)() -> generatedName); - addColumn(new BaseColumnInfo("lsid", JdbcType.VARCHAR), (Supplier)() -> lsidBuilder.setObjectId(String.valueOf(_lsidDbSeq.next())).toString()); + if (!useLsidForUpdate) + { + DbSequence lsidDbSeq = sampleType.getSampleLsidDbSeq(batchSize, sampleType.getContainer()); + addColumn(new BaseColumnInfo("lsid", JdbcType.VARCHAR), (Supplier) () -> lsidBuilder.setObjectId(String.valueOf(lsidDbSeq.next())).toString()); + } addColumn(new BaseColumnInfo("cpasType", JdbcType.VARCHAR), new SimpleTranslator.ConstantColumn(sampleType.getLSID())); addColumn(new BaseColumnInfo("materialSourceId", JdbcType.INTEGER), new SimpleTranslator.ConstantColumn(sampleType.getRowId())); } From d53e9f6ae1b4bdcbc5663f7bad3877d8742eb74d Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 3 Dec 2025 15:34:00 -0800 Subject: [PATCH 32/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index d1503713511..6a3d06f77f2 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.0.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.0.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-KCgJc3ZQv8kUUfNI0pUfJYLwUgt6s/CyhRpnW0s3sPbAl9hQFrEZVbwrADDCXWMXGFhujY2A+AL8a0OFHfymKw==", + "version": "7.1.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-bnoC9ERAEL3uSbJC5ZFf8BbCF4KvPolN9gQ2jgY9IP57P06K5gGwt+IlrZH3inswglMjbHBzZ4oN/dD9T/uLfg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index 32bba39725c..b87cc74fb27 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index e468b5d6e11..0aca74f38f6 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.0.1-fb-remove-sample-lsid.0", + "@labkey/components": "7.1.1-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.0.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.0.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-KCgJc3ZQv8kUUfNI0pUfJYLwUgt6s/CyhRpnW0s3sPbAl9hQFrEZVbwrADDCXWMXGFhujY2A+AL8a0OFHfymKw==", + "version": "7.1.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-bnoC9ERAEL3uSbJC5ZFf8BbCF4KvPolN9gQ2jgY9IP57P06K5gGwt+IlrZH3inswglMjbHBzZ4oN/dD9T/uLfg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 15cf987d66d..69d789ebf5c 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.0.1-fb-remove-sample-lsid.0", + "@labkey/components": "7.1.1-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index f2c7767049c..f63b5fe10dc 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.0.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.0.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-KCgJc3ZQv8kUUfNI0pUfJYLwUgt6s/CyhRpnW0s3sPbAl9hQFrEZVbwrADDCXWMXGFhujY2A+AL8a0OFHfymKw==", + "version": "7.1.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-bnoC9ERAEL3uSbJC5ZFf8BbCF4KvPolN9gQ2jgY9IP57P06K5gGwt+IlrZH3inswglMjbHBzZ4oN/dD9T/uLfg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index 5e3326f2cd4..b9403be6b9f 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index 762787abef2..f9bc40e5bc0 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.0.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.0.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-KCgJc3ZQv8kUUfNI0pUfJYLwUgt6s/CyhRpnW0s3sPbAl9hQFrEZVbwrADDCXWMXGFhujY2A+AL8a0OFHfymKw==", + "version": "7.1.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-bnoC9ERAEL3uSbJC5ZFf8BbCF4KvPolN9gQ2jgY9IP57P06K5gGwt+IlrZH3inswglMjbHBzZ4oN/dD9T/uLfg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index e79ead3b18c..39d26f204ad 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "7.0.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", From 2beaf104519e7e62a2790b3647398af27cc17c2b Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 4 Dec 2025 09:01:29 -0800 Subject: [PATCH 33/62] Address TODOs --- .../SampleUpdateAddColumnsDataIterator.java | 10 +- .../labkey/experiment/ExpDataIterators.java | 23 +-- .../api/SampleTypeUpdateServiceDI.java | 139 +++++++++++------- 3 files changed, 102 insertions(+), 70 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java b/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java index 56883c63969..e4bf7230f05 100644 --- a/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java @@ -5,13 +5,13 @@ import org.labkey.api.collections.Sets; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.CompareType; +import org.labkey.api.data.JdbcType; import org.labkey.api.data.SimpleFilter; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.FieldKey; -import org.labkey.api.util.StringUtilsLabKey; import java.util.LinkedHashMap; import java.util.Map; @@ -113,18 +113,14 @@ protected void prefetchExisting() throws BatchValidationException int rowsToFetch = 50; String keyFieldName = pkColumn.getName(); boolean numericKey = pkColumn.isNumericType(); + JdbcType jdbcType = pkColumn.getJdbcType(); Map rowKeyMap = new LinkedHashMap<>(); Map keyRowMap = new LinkedHashMap<>(); do { lastPrefetchRowNumber = asInteger(_delegate.get(0)); Object keyObj = pkSupplier.get(); - - Object key = null; - if (keyObj instanceof String s) - key = numericKey ? Long.valueOf(s) : StringUtilsLabKey.unquoteString(s); - else if (keyObj instanceof Number) - key = numericKey ? keyObj : keyObj.toString(); + Object key = jdbcType.convert(keyObj); if (numericKey) { diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 1020808353e..a6f0f5b42ec 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -1008,15 +1008,17 @@ protected Set> _getParentParts() return allParts; } - protected void _processRun(ExpRunItem runItem, - List runRecords, - Set> parentNames, - RemapCache cache, - Map materialCache, - Map dataCache, - @Nullable String aliquotedFrom, - String dataType /*sample type or source type name*/, - boolean updateOnly) throws ValidationException, ExperimentException + protected void _processRun( + ExpRunItem runItem, + List runRecords, + Set> parentNames, + RemapCache cache, + Map materialCache, + Map dataCache, + @Nullable String aliquotedFrom, + String dataType /*sample type or source type name*/, + boolean updateOnly + ) throws ValidationException, ExperimentException { Pair pair; if (_context.getInsertOption().allowUpdate) @@ -2500,6 +2502,9 @@ protected SampleNameChangeDataIterator( _nameCol = DataIteratorUtil.createColumnNameMap(di).get(Name.name()); _isAllowUserSpecificNamesValue = NameExpressionOptionService.get().getAllowUserSpecificNamesValue(container); _user = user; + + if (!di.supportsGetExistingRecord()) + throw new IllegalArgumentException("DataIterator must support getExistingRecord()"); } @Override diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 38720da5844..f9b3368b5a8 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -29,8 +29,6 @@ import org.labkey.api.audit.AuditLogService; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.CaseInsensitiveMapWrapper; -import org.labkey.api.collections.IntHashMap; import org.labkey.api.collections.LongHashSet; import org.labkey.api.collections.Sets; import org.labkey.api.data.BaseColumnInfo; @@ -40,6 +38,7 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ConversionExceptionWithMessage; +import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; import org.labkey.api.data.DbSequence; import org.labkey.api.data.Filter; @@ -51,7 +50,9 @@ import org.labkey.api.data.NameGeneratorState; import org.labkey.api.data.RemapCache; import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -81,7 +82,6 @@ import org.labkey.api.exp.api.SampleTypeService; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.PropertyService; import org.labkey.api.exp.query.ExpMaterialTable; import org.labkey.api.exp.query.ExpSchema; import org.labkey.api.exp.query.SamplesSchema; @@ -109,6 +109,7 @@ import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.study.publish.StudyPublishService; import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.util.GUID; import org.labkey.api.util.JobRunner; import org.labkey.api.util.Pair; import org.labkey.api.util.StringUtilsLabKey; @@ -1358,10 +1359,7 @@ public Map> getExistingRows( Set selectColumns = existingRowSelect.columns; Map> sampleRows = new LinkedHashMap<>(); - Map rowNumLsid = new IntHashMap<>(); - Map rowIdRowNumMap = new LinkedHashMap<>(); - Map lsidRowNumMap = new CaseInsensitiveMapWrapper<>(new LinkedHashMap<>()); Map nameRowNumMap = new LinkedHashMap<>(); Integer sampleTypeId = null; for (Map.Entry> keyMap : keys.entrySet()) @@ -1374,14 +1372,6 @@ public Map> getExistingRows( continue; } - String lsid = getMaterialLsid(keyMap.getValue()); - if (lsid != null) - { - lsidRowNumMap.put(lsid, rowNum); - rowNumLsid.put(rowNum, lsid); - continue; - } - String name = getMaterialName(keyMap.getValue()); Integer materialSourceId = getMaterialSourceId(keyMap.getValue()); if (name != null && materialSourceId != null) @@ -1403,37 +1393,17 @@ public Map> getExistingRows( { Long rowId = asLong(row.get(RowId.name())); Integer rowNum = rowIdRowNumMap.get(rowId); - String sampleLsid = (String) row.get(LSID.name()); - - rowNumLsid.put(rowNum, sampleLsid); sampleRows.put(rowNum, row); } } - Set allKeys = new HashSet<>(); - boolean useLsid = false; + Set allKeys; - if (!lsidRowNumMap.isEmpty()) - { - useLsid = true; - allKeys.addAll(lsidRowNumMap.keySet()); - - SimpleFilter filter = new SimpleFilter(LSID.fieldKey(), lsidRowNumMap.keySet(), CompareType.IN); - filter.addCondition(FieldKey.fromParts("Container"), container); - Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); - for (Map row : rows) - { - String sampleLsid = (String) row.get(LSID.name()); - Integer rowNum = lsidRowNumMap.get(sampleLsid); - sampleRows.put(rowNum, row); - - allKeys.remove(sampleLsid); - } - } - - if (!nameRowNumMap.isEmpty()) + if (nameRowNumMap.isEmpty()) + allKeys = Collections.emptySet(); + else { - allKeys.addAll(nameRowNumMap.keySet()); + allKeys = new HashSet<>(nameRowNumMap.keySet()); SimpleFilter filter = new SimpleFilter(MaterialSourceId.fieldKey(), sampleTypeId); filter.addCondition(Name.fieldKey(), nameRowNumMap.keySet(), CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), container); @@ -1442,29 +1412,32 @@ public Map> getExistingRows( { String name = (String) row.get(Name.name()); Integer rowNum = nameRowNumMap.get(name); - String sampleLsid = (String) row.get(LSID.name()); sampleRows.put(rowNum, row); - rowNumLsid.put(rowNum, sampleLsid); - allKeys.remove(name); } } if (verifyNoCrossFolderData && !allKeys.isEmpty()) { - // Issue 52922: cross folder merge without Product Folders enabled silently ignores the cross folder row update - ContainerFilter allCf = new ContainerFilter.AllInProjectPlusShared(container, user); // use a relaxed CF to find existing data from cross containers - - SimpleFilter existingDataFilter = new SimpleFilter(MaterialSourceId.fieldKey(), sampleTypeId); - existingDataFilter.addCondition(allCf.createFilterClause(ExperimentService.get().getSchema(), FieldKey.fromParts("Container"))); - existingDataFilter.addCondition(useLsid ? LSID.fieldKey() : Name.fieldKey(), allKeys, CompareType.IN); + // Issue 52922: cross-folder merge without Product Folders enabled silently ignores the cross-folder + // row update. Use a relaxed container filter to find existing data from cross-containers. + ContainerFilter cf = new ContainerFilter.AllInProjectPlusShared(container, user); + Set containerIds = new HashSet<>(Objects.requireNonNull(cf.getIds())); + containerIds.remove(container.getEntityId()); - // TODO: Couldn't this question be asked in the query and return a max of one row where the container does not match? - Map[] cfRows = new TableSelector(ExperimentService.get().getTinfoMaterial(), Sets.newCaseInsensitiveHashSet("Container", Name.name()), existingDataFilter, null).getMapArray(); - for (Map row : cfRows) + if (!containerIds.isEmpty()) { - String dataContainer = (String) row.get("container"); - if (!dataContainer.equals(container.getId())) + TableInfo table = ExperimentService.get().getTinfoMaterial(); + DbSchema schema = table.getSchema(); + + SQLFragment sql = new SQLFragment("SELECT Name FROM ").append(table) + .append(" WHERE MaterialSourceId = ?").add(sampleTypeId) + .append(" AND Name ").appendInClause(allKeys, schema.getSqlDialect()) + .append(" AND Container ").appendInClause(containerIds, schema.getSqlDialect()) + .append(" LIMIT 1"); + + var row = new SqlSelector(schema, sql).getMap(); + if (row != null) throw new InvalidKeyException("Sample does not belong to " + container.getName() + " container: " + row.get("name") + "."); } } @@ -1508,7 +1481,12 @@ public List> getRows(User user, Container container, List> result = new ArrayList<>(keys.size()); for (Map k : keys) { @@ -1519,6 +1497,59 @@ public List> getRows(User user, Container container, List> keys) throws QueryUpdateServiceException + { + List rowIds = new ArrayList<>(); + List lsids = new ArrayList<>(); + Map> namesBySourceId = new HashMap<>(); + int nameCount = 0; + + // Each row could be keyed differently + for (Map row : keys) + { + Long rowId = getMaterialRowId(row); + if (rowId != null) + { + rowIds.add(rowId); + continue; + } + + String lsid = getMaterialLsid(row); + if (lsid != null) + { + lsids.add(lsid); + continue; + } + + String name = getMaterialName(row); + Integer materialSourceId = getMaterialSourceId(row); + if (name != null && materialSourceId != null) + { + namesBySourceId.computeIfAbsent(materialSourceId, k -> new ArrayList<>()).add(name); + nameCount++; + continue; + } + + throw new QueryUpdateServiceException("Either RowId, LSID, or Name and MaterialSourceId is required to get Sample Type Material."); + } + + // But we can optimize if they all share the same filter + SimpleFilter filter = null; + if (rowIds.size() == keys.size()) + filter = new SimpleFilter(RowId.fieldKey(), rowIds, CompareType.IN); + else if (lsids.size() == keys.size()) + filter = new SimpleFilter(LSID.fieldKey(), lsids, CompareType.IN); + else if (nameCount == keys.size() && namesBySourceId.size() == 1) + { + // If all rows are being queried by name and share the same material source id, use a single filter + Map.Entry> entry = namesBySourceId.entrySet().iterator().next(); + filter = new SimpleFilter(MaterialSourceId.fieldKey(), entry.getKey()); + filter.addCondition(Name.fieldKey(), entry.getValue(), CompareType.IN); + } + + return filter; + } + @Override protected Map getRow(User user, Container container, Map keys) throws QueryUpdateServiceException { From e0c4c1844f137453d1d58520c3b186ff0ab6ddba Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 4 Dec 2025 10:01:04 -0800 Subject: [PATCH 34/62] Use TableSelector --- .../api/SampleTypeUpdateServiceDI.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index f9b3368b5a8..febd089de0f 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -38,7 +38,6 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ConversionExceptionWithMessage; -import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; import org.labkey.api.data.DbSequence; import org.labkey.api.data.Filter; @@ -50,9 +49,7 @@ import org.labkey.api.data.NameGeneratorState; import org.labkey.api.data.RemapCache; import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlSelector; import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -1427,16 +1424,11 @@ public Map> getExistingRows( if (!containerIds.isEmpty()) { - TableInfo table = ExperimentService.get().getTinfoMaterial(); - DbSchema schema = table.getSchema(); + SimpleFilter filter = new SimpleFilter(MaterialSourceId.fieldKey(), sampleTypeId); + filter.addCondition(FieldKey.fromParts("Container"), containerIds, CompareType.IN); + filter.addCondition(Name.fieldKey(), allKeys, CompareType.IN); - SQLFragment sql = new SQLFragment("SELECT Name FROM ").append(table) - .append(" WHERE MaterialSourceId = ?").add(sampleTypeId) - .append(" AND Name ").appendInClause(allKeys, schema.getSqlDialect()) - .append(" AND Container ").appendInClause(containerIds, schema.getSqlDialect()) - .append(" LIMIT 1"); - - var row = new SqlSelector(schema, sql).getMap(); + var row = new TableSelector(ExperimentService.get().getTinfoMaterial(), Sets.newCaseInsensitiveHashSet(Name.name()), filter, null).setMaxRows(1).getMap(); if (row != null) throw new InvalidKeyException("Sample does not belong to " + container.getName() + " container: " + row.get("name") + "."); } From cbc078d55c20b77e27a728789774817cfb082ebb Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 4 Dec 2025 11:52:33 -0800 Subject: [PATCH 35/62] No longer accept LSID for update --- .../api/SampleTypeUpdateServiceDI.java | 105 +++++++++--------- 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index febd089de0f..57f41d3cce7 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -530,16 +530,7 @@ private static boolean useDataIteratorForUpdate( if (rows == null || rows.isEmpty() || oldKeys != null) return false; - Set columnNames = rows.get(0).keySet(); - - // For backwards compatibility we continue to allow specification of an LSID as a key iff RowId is not provided. - // This is only supported via row-by-row update. - boolean useDib = !(columnNames.contains(LSID.name()) && !columnNames.contains(RowId.name())); - - // All rows must have a uniform set of keys for the data iterator to work. - useDib = useDib && hasUniformKeys(rows); - - return useDib; + return hasUniformKeys(rows); } @Override @@ -866,6 +857,9 @@ protected void validateUpdateRow(Map row) throws ValidationExcep protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, boolean allowOwner, boolean retainCreation) throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException { + if (row.containsKey(LSID.name()) && !(row.containsKey(RowId.name()) || row.containsKey(Name.name()))) + throw new ValidationException("Either RowId or Name is required to update a sample."); + Map result = super.updateRow(user, container, row, oldRow, allowOwner, retainCreation); // add MaterialInput/DataInputs field from parent alias @@ -1183,9 +1177,9 @@ public List> deleteRows(User user, Container container, List return null; } - private @Nullable Integer getMaterialSourceId(Map row) + private @Nullable Long getMaterialSourceId(Map row) { - return getMaterialIntegerValue(row, MaterialSourceId.name()); + return MapUtils.getLong(row, MaterialSourceId.name()); } private @Nullable Long getMaterialRowId(Map row) @@ -1193,7 +1187,7 @@ public List> deleteRows(User user, Container container, List return MapUtils.getLong(row, RowId.name()); } - private @Nullable Filter getMaterialFilter(Map keys) + private @Nullable Filter getMaterialFilter(Map keys, boolean useSampleType) { Long rowId = getMaterialRowId(keys); if (rowId != null) @@ -1204,12 +1198,23 @@ public List> deleteRows(User user, Container container, List return new SimpleFilter(LSID.fieldKey(), lsid); String name = getMaterialName(keys); - Integer materialSourceId = getMaterialSourceId(keys); - if (name != null && materialSourceId != null) + if (name != null) { - SimpleFilter filter = new SimpleFilter(Name.fieldKey(), name); - filter.addCondition(MaterialSourceId.fieldKey(), materialSourceId); - return filter; + Long materialSourceId = null; + if (useSampleType) + { + if (_sampleType != null) + materialSourceId = _sampleType.getRowId(); + } + else + materialSourceId = getMaterialSourceId(keys); + + if (materialSourceId != null) + { + SimpleFilter filter = new SimpleFilter(Name.fieldKey(), name); + filter.addCondition(MaterialSourceId.fieldKey(), materialSourceId); + return filter; + } } return null; @@ -1217,46 +1222,22 @@ public List> deleteRows(User user, Container container, List private Map getMaterialMap(Map keys) throws QueryUpdateServiceException { - Filter filter = getMaterialFilter(keys); - if (filter == null) - throw new QueryUpdateServiceException("Either RowId, LSID, or Name and MaterialSourceId is required to get Sample Type Material."); - - return new TableSelector(getQueryTable(), filter, null).getMap(); + return getMaterialMap(keys, false); } - private @Nullable Map getMaterialMapWithInputs( - Long rowId, - String lsid, - User user, - Container container - ) throws QueryUpdateServiceException + private Map getMaterialMap(Map keys, boolean useSampleType) throws QueryUpdateServiceException { - Filter filter; - if (rowId != null) - filter = new SimpleFilter(RowId.fieldKey(), rowId); - else if (lsid != null) - filter = new SimpleFilter(LSID.fieldKey(), lsid); - else - throw new QueryUpdateServiceException("Either RowId or LSID is required to get Sample Type Material."); - - Map sampleRow = new TableSelector(getQueryTable(), filter, null).getMap(); - if (null == sampleRow) - return sampleRow; - - ExperimentService experimentService = ExperimentService.get(); - ExpMaterial seed = rowId != null ? experimentService.getExpMaterial(rowId) : experimentService.getExpMaterial(lsid); - if (null == seed) - return sampleRow; - - ExperimentServiceImpl.get().addParentsFields(seed, sampleRow, user, container); + Filter filter = getMaterialFilter(keys, useSampleType); + if (filter == null) + throw new QueryUpdateServiceException("Either RowId, LSID, or Name and MaterialSourceId is required to get sample."); - return sampleRow; + return new TableSelector(getQueryTable(), filter, null).getMap(); } @Override public boolean hasExistingRowsInOtherContainers(Container container, Map> keys) { - Integer sampleTypeId = null; + Long sampleTypeId = null; Set sampleNames = new HashSet<>(); for (Map.Entry> keyMap : keys.entrySet()) { @@ -1358,7 +1339,7 @@ public Map> getExistingRows( Map> sampleRows = new LinkedHashMap<>(); Map rowIdRowNumMap = new LinkedHashMap<>(); Map nameRowNumMap = new LinkedHashMap<>(); - Integer sampleTypeId = null; + Long sampleTypeId = null; for (Map.Entry> keyMap : keys.entrySet()) { Integer rowNum = keyMap.getKey(); @@ -1370,7 +1351,7 @@ public Map> getExistingRows( } String name = getMaterialName(keyMap.getValue()); - Integer materialSourceId = getMaterialSourceId(keyMap.getValue()); + Long materialSourceId = getMaterialSourceId(keyMap.getValue()); if (name != null && materialSourceId != null) { sampleTypeId = materialSourceId; @@ -1493,7 +1474,7 @@ public List> getRows(User user, Container container, List rowIds = new ArrayList<>(); List lsids = new ArrayList<>(); - Map> namesBySourceId = new HashMap<>(); + Map> namesBySourceId = new HashMap<>(); int nameCount = 0; // Each row could be keyed differently @@ -1514,7 +1495,7 @@ public List> getRows(User user, Container container, List new ArrayList<>()).add(name); @@ -1534,7 +1515,7 @@ else if (lsids.size() == keys.size()) else if (nameCount == keys.size() && namesBySourceId.size() == 1) { // If all rows are being queried by name and share the same material source id, use a single filter - Map.Entry> entry = namesBySourceId.entrySet().iterator().next(); + Map.Entry> entry = namesBySourceId.entrySet().iterator().next(); filter = new SimpleFilter(MaterialSourceId.fieldKey(), entry.getKey()); filter.addCondition(Name.fieldKey(), entry.getValue(), CompareType.IN); } @@ -1545,7 +1526,21 @@ else if (nameCount == keys.size() && namesBySourceId.size() == 1) @Override protected Map getRow(User user, Container container, Map keys) throws QueryUpdateServiceException { - return getMaterialMapWithInputs(getMaterialRowId(keys), getMaterialLsid(keys), user, container); + Map sampleRow = getMaterialMap(keys, true); + if (sampleRow == null) + return null; + + Long sampleRowId = asLong(sampleRow.get(RowId.name())); + if (sampleRowId == null) + throw new QueryUpdateServiceException("Failed to resolve sample rowId."); + + ExpMaterial seed = ExperimentService.get().getExpMaterial(sampleRowId); + if (null == seed) + return sampleRow; + + ExperimentServiceImpl.get().addParentsFields(seed, sampleRow, user, container); + + return sampleRow; } private void onSamplesChanged(List> results, Map params, Container container, SampleTypeServiceImpl.SampleChangeType reason) From 13a68d1188db741b446292ab4b81ec343b67243f Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 4 Dec 2025 16:23:13 -0800 Subject: [PATCH 36/62] Add back property column --- .../experiment/api/ExpMaterialTableImpl.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index a23e9ffc98d..db5d6750395 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -688,6 +688,26 @@ protected ContainerFilter getLookupContainerFilter() } } + @Override + public MutableColumnInfo createPropertyColumn(String alias) + { + var ret = wrapColumn(alias, _rootTable.getColumn(RowId.name())); + + if (_ss != null && _ss.getTinfo() != null) + { + ForeignKey fk = new QueryForeignKey.Builder(getUserSchema(), getLookupContainerFilter()) + .table(_ss.getTinfo()) + .key(RowId.name()) + .build(); + ret.setFk(fk); + } + + ret.setIsUnselectable(true); + ret.setDescription("A holder for any custom fields associated with this sample"); + ret.setHidden(true); + return ret; + } + private static boolean isStatusEnabled(Container c) { return SampleStatusService.get().supportsSampleStatus() && !SampleStatusService.get().getAllProjectStates(c).isEmpty(); From 309d2d329f2b6102e1723e60aae1c491a0d74cfe Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 5 Dec 2025 14:10:44 -0800 Subject: [PATCH 37/62] Disallow rowId when merging --- .../labkey/experiment/ExpDataIterators.java | 100 ++++-- .../experiment/api/ExpSampleTypeTestCase.jsp | 297 +++++++++++------- .../api/SampleTypeUpdateServiceDI.java | 18 +- 3 files changed, 277 insertions(+), 138 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index a6f0f5b42ec..64a3dc907ee 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -531,6 +531,9 @@ protected AliasDataIterator(DataIterator di, DataIteratorContext context, Contai _nameCol = map.get(Name.name()) == null ? null : di.getSupplier(map.get(Name.name())); _expAliasTable = expAliasTable; _isUpdateOnly = _context.getInsertOption().updateOnly; + + if (_isUpdateOnly && !di.supportsGetExistingRecord()) + throw new IllegalArgumentException("DataIterator must support getExistingRecord() to update aliases"); } @Override @@ -752,6 +755,7 @@ private static class FlagDataIterator extends ExpDataTypeDataIterator final Integer _lsidCol; final Integer _nameCol; final Integer _flagCol; + final boolean _isUpdateOnly; protected FlagDataIterator(DataIterator di, DataIteratorContext context, User user, boolean isSample, ExpObject dataType, Container container) { @@ -762,6 +766,10 @@ protected FlagDataIterator(DataIterator di, DataIteratorContext context, User us _lsidCol = map.get("lsid"); _nameCol = map.get("name"); _flagCol = map.containsKey("flag") ? map.get("flag") : map.get("comment"); + _isUpdateOnly = _context.getInsertOption().updateOnly; + + if (_isUpdateOnly && !di.supportsGetExistingRecord()) + throw new IllegalArgumentException("DataIterator must support getExistingRecord() to update flag/comment"); } @Override @@ -779,7 +787,7 @@ public boolean next() throws BatchValidationException return true; ExpObject expObject = null; - if (_nameCol != null && (_context.getInsertOption().mergeRows || _context.getInsertOption().updateOnly)) + if (_nameCol != null && (_context.getInsertOption().mergeRows || _isUpdateOnly)) { Object nameValue = get(_nameCol); if (nameValue instanceof String name) @@ -793,6 +801,17 @@ public boolean next() throws BatchValidationException expObject = getExpObjectByLsid(lsid); } + if (expObject == null && _isUpdateOnly) + { + Map oldRow = getExistingRecord(); + if (oldRow != null) + { + String lsid = (String) oldRow.get(LSID.name()); + if (lsid != null) + expObject = getExpObjectByLsid(lsid); + } + } + if (expObject != null) { Object flagValue = get(_flagCol); @@ -1108,7 +1127,7 @@ protected void _processRun( } } - static class DerivationDataIterator extends DerivationDataIteratorBase + private static class DerivationDataIterator extends DerivationDataIteratorBase { final Integer _aliquotParentCol; final Map _lsidNames; @@ -1273,7 +1292,7 @@ else if (!_skipAliquot && _context.getInsertOption().mergeRows) } } - static class SampleUpdateDerivationDataIterator extends DerivationDataIteratorBase + private static class SampleUpdateDerivationDataIterator extends DerivationDataIteratorBase { final Integer _aliquotParentCol; // Map from Data name to Set of (parentColName, parentName) final Map _aliquotParents; // Map of Data name and its aliquotedFromLSID @@ -1407,7 +1426,7 @@ else if (o instanceof Number) } } - static class DataUpdateDerivationDataIterator extends DerivationDataIteratorBase + private static class DataUpdateDerivationDataIterator extends DerivationDataIteratorBase { // Map from Data name to Set of (parentColName, parentName) final Map>> _parentNames; @@ -2113,6 +2132,9 @@ protected SearchIndexIterator(DataIterator di, DataIteratorContext context, Func _rowIds = new LongArrayList(100); _isInsert = !context.getInsertOption().allowUpdate; // only useRowIdCol for INSERT. For UPDATE, rowId usually is not available. For MERGE, rowId is a new DBSequence value for existing data + + if (!_isInsert && !di.supportsGetExistingRecord()) + throw new IllegalArgumentException("DataIterator must support getExistingRecord() for search index update."); } static Long asLong(Object o) @@ -2391,10 +2413,15 @@ public DataIterator getDataIterator(DataIteratorContext context) dib = getRootMaterialRowIdBuilder(dib); if (isMergeOrUpdate) + { dib = new SampleStatusCheckIteratorBuilder(dib, _container); - if (isUpdateOnly) - dib = new SampleUpdateOnlyDataIteratorBuilder(dib, context, _container, _user); + if (isUpdateOnly) + { + dib = new SampleUpdateOnlyValidatorsIteratorBuilder(dib, _container, _user); + dib = new SampleNameChangeDataIteratorBuilder(dib, _user, canUpdateNames); + } + } } // Insert into exp.data then the provisioned table @@ -2445,19 +2472,17 @@ private DataIteratorBuilder getRootMaterialRowIdBuilder(DataIteratorBuilder dib) } } - private static class SampleUpdateOnlyDataIteratorBuilder implements DataIteratorBuilder + private static class SampleUpdateOnlyValidatorsIteratorBuilder implements DataIteratorBuilder { private final Container _container; private final DataIteratorBuilder _in; private final User _user; - public SampleUpdateOnlyDataIteratorBuilder(@NotNull DataIteratorBuilder in, DataIteratorContext context, Container container, User user) + public SampleUpdateOnlyValidatorsIteratorBuilder(@NotNull DataIteratorBuilder in, Container container, User user) { _container = container; _in = in; _user = user; - - assert context.getInsertOption().updateOnly : "SampleUpdateOnlyDataIteratorBuilder should only be used for UPDATE_ONLY"; } @Override @@ -2479,7 +2504,28 @@ public DataIterator getDataIterator(DataIteratorContext context) if (validate.hasValidators()) di = validate; - return LoggingDataIterator.wrap(new SampleNameChangeDataIterator(di, context, _container, _user)); + return LoggingDataIterator.wrap(di); + } + } + + private static class SampleNameChangeDataIteratorBuilder implements DataIteratorBuilder + { + private final DataIteratorBuilder _in; + private final boolean _canUpdateNames; + private final User _user; + + public SampleNameChangeDataIteratorBuilder(@NotNull DataIteratorBuilder in, User user, boolean canUpdateNames) + { + _in = in; + _canUpdateNames = canUpdateNames; + _user = user; + } + + @Override + public DataIterator getDataIterator(DataIteratorContext context) + { + DataIterator di = _in.getDataIterator(context); + return LoggingDataIterator.wrap(new SampleNameChangeDataIterator(di, context, _user, _canUpdateNames)); } } @@ -2487,20 +2533,20 @@ private static class SampleNameChangeDataIterator extends WrapperDataIterator { private final DataIteratorContext _context; private final Integer _nameCol; - private final boolean _isAllowUserSpecificNamesValue; + private final boolean _canUpdateNames; private final User _user; protected SampleNameChangeDataIterator( DataIterator di, DataIteratorContext context, - Container container, - User user + User user, + boolean canUpdateNames ) { super(di); _context = context; _nameCol = DataIteratorUtil.createColumnNameMap(di).get(Name.name()); - _isAllowUserSpecificNamesValue = NameExpressionOptionService.get().getAllowUserSpecificNamesValue(container); + _canUpdateNames = canUpdateNames; _user = user; if (!di.supportsGetExistingRecord()) @@ -2514,27 +2560,29 @@ public boolean next() throws BatchValidationException if (!hasNext) return false; + if (_nameCol == null || _context.getErrors().hasErrors()) + return true; + var existingRecord = getExistingRecord(); - if (_nameCol == null || _context.getErrors().hasErrors() || existingRecord == null) + if (existingRecord == null) return true; Object newNameObj = get(_nameCol); String newName = newNameObj == null ? null : String.valueOf(newNameObj); String oldName = (String) existingRecord.get(Name.name()); boolean hasNameChange = !StringUtils.isEmpty(newName) && !newName.equals(oldName); + if (!hasNameChange) + return true; - if (hasNameChange) + if (_canUpdateNames) { - if (_isAllowUserSpecificNamesValue) - { - Long rowId = asLong(existingRecord.get(RowId.name())); - var sample = ExperimentService.get().getExpMaterial(rowId); - if (sample != null) - ExperimentService.get().addObjectLegacyName(sample.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpMaterial.class), oldName, _user); - } - else - _context.getErrors().addRowError(new ValidationException("User-specified sample name not allowed")); + Long rowId = asLong(existingRecord.get(RowId.name())); + ExpMaterial sample = ExperimentService.get().getExpMaterial(rowId); + if (sample != null) + ExperimentService.get().addObjectLegacyName(sample.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpMaterial.class), oldName, _user); } + else + _context.getErrors().addRowError(new ValidationException("User-specified sample name not allowed")); return true; } diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index f8ce588465b..c2c85b29a5a 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -50,14 +50,11 @@ <%@ page import="org.labkey.api.gwt.client.AuditBehaviorType" %> <%@ page import="org.labkey.api.gwt.client.model.GWTPropertyDescriptor" %> <%@ page import="org.labkey.api.query.BatchValidationException" %> -<%@ page import="org.labkey.api.query.DefaultSchema" %> <%@ page import="org.labkey.api.query.FieldKey" %> -<%@ page import="org.labkey.api.query.QuerySchema" %> <%@ page import="org.labkey.api.query.QueryService" %> <%@ page import="org.labkey.api.query.QueryUpdateService" %> <%@ page import="static org.hamcrest.CoreMatchers.hasItems" %> <%@ page import="static org.junit.Assert.*" %> -<%@ page import="org.labkey.api.query.SchemaKey" %> <%@ page import="org.labkey.api.query.UserSchema" %> <%@ page import="org.labkey.api.reader.DataLoader" %> <%@ page import="org.labkey.api.reader.TabLoader" %> @@ -308,7 +305,7 @@ public void idColsSet_nameExpressionNull_hasUnusedNameProperty() throws Exceptio props.add(new GWTPropertyDescriptor("name", "string")); props.add(new GWTPropertyDescriptor("age", "int")); - final ExpSampleType st = SampleTypeService.get().createSampleType(c, user, + SampleTypeService.get().createSampleType(c, user, "Samples", null, props, Collections.emptyList(), 1, -1, -1, -1, null, null); fail("Expected exception"); @@ -340,7 +337,7 @@ public void idColsSet_nameExpressionNull_hasNameProperty() throws Exception rows.add(CaseInsensitiveHashMap.of("name", "bob", "prop", "blue", "age", 10)); BatchValidationException errors = new BatchValidationException(); - QueryUpdateService svc = getSampleTypeUpdateService("Samples"); + QueryUpdateService svc = getSampleTypeUpdateService(st.getName()); svc.insertRows(user, c, rows, errors, null, null); if (errors.hasErrors()) throw errors; @@ -369,7 +366,7 @@ public void idColsSet_nameExpression_hasUnusedNameProperty() throws Exception props.add(new GWTPropertyDescriptor("name", "string")); props.add(new GWTPropertyDescriptor("age", "int")); - final ExpSampleType st = SampleTypeService.get().createSampleType(c, user, + SampleTypeService.get().createSampleType(c, user, "Samples", null, props, Collections.emptyList(), 1, -1, -1, -1, "S-${name}.${age}", null); fail("Expected exception"); @@ -507,27 +504,109 @@ public void testNameExpression() throws Exception @Test public void testAliases() throws Exception { - // setup List props = new ArrayList<>(); props.add(new GWTPropertyDescriptor("name", "string")); props.add(new GWTPropertyDescriptor("age", "int")); - final ExpSampleType st = createSampleType("Samples", props, null); - List> rows = new ArrayList<>(); - Map row = new CaseInsensitiveHashMap<>(); - row.put("name", "boo"); - row.put("age", 20); - row.put("alias", "a,b,c"); - rows.add(row); + ExpMaterial fooSample; + ExpMaterial booSample; + List> rows; - insertSampleRows("Samples", rows); + // Insert + { + rows = new ArrayList<>(); + rows.add(CaseInsensitiveHashMap.of("name", "foo", "age", 19, "alias", "youth, teen, minor", "flag", "new zealand")); + rows.add(CaseInsensitiveHashMap.of("name", "boo", "age", 21, "alias", "elder, adult, major?", "flag", "australia")); - ExpMaterial m = st.getSample(c, "boo"); - Collection aliases = m.getAliases(); - assertThat(aliases, hasItems("a", "b", "c")); -} + insertSampleRows(st.getName(), rows); + + fooSample = st.getSample(c, "foo"); + assertThat(fooSample.getAliases(), hasItems("youth", "teen", "minor")); + assertEquals("new zealand", fooSample.getComment()); + + booSample = st.getSample(c, "boo"); + assertThat(booSample.getAliases(), hasItems("elder", "adult", "major?")); + assertEquals("australia", booSample.getComment()); + } + + // Update, keyed by name + { + rows = new ArrayList<>(); + rows.add(CaseInsensitiveHashMap.of("name", fooSample.getName(), "alias", "gerald, r, ford", "flag", "kenya")); + rows.add(CaseInsensitiveHashMap.of("name", booSample.getName(), "alias", "dwight, d, eisenhower", "flag", "uganda")); + updateSampleRows(st.getName(), rows); + + fooSample = st.getSample(c, fooSample.getName()); + assertThat(fooSample.getAliases(), hasItems("gerald", "r", "ford")); + assertEquals("kenya", fooSample.getComment()); + + booSample = st.getSample(c, "boo"); + assertThat(booSample.getAliases(), hasItems("dwight", "d", "eisenhower")); + assertEquals("uganda", booSample.getComment()); + } + + // Update, keyed by rowId + { + rows = new ArrayList<>(); + rows.add(CaseInsensitiveHashMap.of("rowId", fooSample.getRowId(), "alias", "ken, griffey", "flag", "norway")); + rows.add(CaseInsensitiveHashMap.of("rowId", booSample.getRowId(), "alias", "edgar, martinez", "flag", "sweden")); + + updateSampleRows(st.getName(), rows); + + fooSample = st.getSample(c, fooSample.getName()); + assertThat(fooSample.getAliases(), hasItems("ken", "griffey")); + assertEquals("norway", fooSample.getComment()); + + booSample = st.getSample(c, booSample.getName()); + assertThat(booSample.getAliases(), hasItems("edgar", "martinez")); + assertEquals("sweden", booSample.getComment()); + } + + // Update with different row shapes + { + rows = new ArrayList<>(); + rows.add(CaseInsensitiveHashMap.of("rowId", fooSample.getRowId(), "description", "east coast destination", "alias", "martha's, vineyard", "flag", "japan")); + rows.add(CaseInsensitiveHashMap.of("name", booSample.getName(), "age", 1_000_000, "alias", "cannon, beach", "flag", "fiji")); + + updateSampleRows(st.getName(), rows); + + fooSample = st.getSample(c, fooSample.getName()); + assertThat(fooSample.getAliases(), hasItems("martha's", "vineyard")); + assertEquals("japan", fooSample.getComment()); + + booSample = st.getSample(c, booSample.getName()); + assertThat(booSample.getAliases(), hasItems("cannon", "beach")); + assertEquals("fiji", booSample.getComment()); + } + + // Merge + { + rows = new ArrayList<>(); + rows.add(CaseInsensitiveHashMap.of("name", fooSample.getName(), "age", 40, "alias", "son, family, father", "flag", "canada")); + rows.add(CaseInsensitiveHashMap.of("name", booSample.getName(), "age", 80, "alias", "grandma, family, mother", "flag", "america")); + rows.add(CaseInsensitiveHashMap.of("name", "moo", "age", 12, "alias", "cow, bovine", "flag", "mexico")); + + mergeSampleRows(st.getName(), rows); + + var expectedRowId = fooSample.getRowId(); + fooSample = st.getSample(c, fooSample.getName()); + assertEquals(expectedRowId, fooSample.getRowId()); + assertThat(fooSample.getAliases(), hasItems("son", "family", "father")); + assertEquals("canada", fooSample.getComment()); + + expectedRowId = booSample.getRowId(); + booSample = st.getSample(c, booSample.getName()); + assertEquals(expectedRowId, booSample.getRowId()); + assertThat(booSample.getAliases(), hasItems("grandma", "family", "mother")); + assertEquals("america", booSample.getComment()); + + ExpMaterial mooSample = st.getSample(c, "moo"); + assertThat(mooSample.getAliases(), hasItems("cow", "bovine")); + assertEquals("mexico", mooSample.getComment()); + } +} // Issue 33682: Calling insertRows on SampleType with empty values will not insert new samples @Test @@ -545,14 +624,8 @@ public void testBlankRows() throws Exception List allSamples = st.getSamples(c); assertTrue("Expected no samples", allSamples.isEmpty()); - // // insert via insertRows -- blank rows should be preserved - // - - UserSchema schema = QueryService.get().getUserSchema(user, c, SchemaKey.fromParts("Samples")); - TableInfo table = schema.getTable("Samples"); - QueryUpdateService svc = table.getUpdateService(); - assertNotNull(svc); + QueryUpdateService svc = getSampleTypeUpdateService(st.getName()); // insert 3 rows with no values List> rows = new ArrayList<>(); @@ -574,10 +647,7 @@ public void testBlankRows() throws Exception assertEquals("Expected 3 total samples", 3, allSamples.size()); assertEquals(0, allSamples.get(0).getAliquotCount()); - // // insert as if we pasted a tsv in the "upload samples" page -- blank rows should be skipped - // - // data has three lines, one blank. expect to insert only two samples String dataTxt = "age\n" + @@ -596,14 +666,16 @@ public void testBlankRows() throws Exception assertEquals(0, insertedRows.get(0).get("AliquotCount")); ExpMaterial material1 = ExperimentService.get().getExpMaterial(asLong(insertedRows.get(0).get("rowid"))); + assertNotNull(material1); Map map = material1.getPropertyValues(); assertEquals("Expected to only have 'age' property, got: " + map, 1, map.size()); - Integer age1 = (Integer)material1.getPropertyValues().values().iterator().next(); + Integer age1 = asInteger(material1.getPropertyValues().values().iterator().next()); assertNotNull(age1); assertEquals("Expected to insert age of 20, got: " + age1, 20, age1.intValue()); - ExpMaterial material2 = ExperimentService.get().getExpMaterial((Integer)insertedRows.get(1).get("rowid")); + ExpMaterial material2 = ExperimentService.get().getExpMaterial(asLong(insertedRows.get(1).get("rowid"))); + assertNotNull(material2); Integer age2 = asInteger(material2.getPropertyValues().values().iterator().next()); assertNotNull(age2); assertEquals("Expected to insert age of 30, got: " + age2, 30, age2.intValue()); @@ -618,7 +690,8 @@ public void testBlankRows() throws Exception updated.put("age", age1 + 1); svc.updateRows(user, c, Collections.singletonList(updated), null, errors, null, null); assertFalse(errors.hasErrors()); - var result = new TableSelector(table, TableSelector.ALL_COLUMNS, new SimpleFilter("lsid", material1.getLSID()), null).getMap(); + TableInfo table = getSampleTypeTable(st.getName()); + var result = new TableSelector(table, new SimpleFilter("lsid", material1.getLSID()), null).getMap(); assertEquals(21, asInteger(result.get("age")).intValue()); // and a delete @@ -641,39 +714,33 @@ public void testUpdateSomeParents() throws Exception final ExpSampleType parent1Type = createSampleType("Parent1Samples", props, null); final ExpSampleType parent2Type = createSampleType("Parent2Samples", props, null); - UserSchema schema = QueryService.get().getUserSchema(user, c, SchemaKey.fromParts("Samples")); - List> rows = new ArrayList<>(); - - // add first parents - TableInfo table = schema.getTable("Parent1Samples", null, true, false); - QueryUpdateService svcParent = table.getUpdateService(); + QueryUpdateService updateService = getSampleTypeUpdateService(parent1Type.getName()); BatchValidationException errors = new BatchValidationException(); + List> rows = new ArrayList<>(); rows.add(CaseInsensitiveHashMap.of("name", "P1-1")); rows.add(CaseInsensitiveHashMap.of("name", "P1-2")); rows.add(CaseInsensitiveHashMap.of("name", "P1-3")); rows.add(CaseInsensitiveHashMap.of("name", "P1-4,test")); - List> inserted = svcParent.insertRows(user, c, rows, errors, null, null); + List> inserted = updateService.insertRows(user, c, rows, errors, null, null); assertFalse(errors.hasErrors()); assertEquals("Number of parent1 samples inserted not as expected", 4, inserted.size()); // add second parents - table = schema.getTable("Parent2Samples", null, true, false); - QueryUpdateService svcParent2 = table.getUpdateService(); + updateService = getSampleTypeUpdateService(parent2Type.getName()); rows.clear(); rows.add(CaseInsensitiveHashMap.of("name", "P2-1")); rows.add(CaseInsensitiveHashMap.of("name", "P2-2")); rows.add(CaseInsensitiveHashMap.of("name", "P2-3")); - inserted = svcParent2.insertRows(user, c, rows, errors, null, null); + inserted = updateService.insertRows(user, c, rows, errors, null, null); assertFalse(errors.hasErrors()); assertEquals("Number of parent2 samples inserted not as expected", 3, inserted.size()); // add child samples - table = schema.getTable("ChildSamples", null, true, false); - QueryUpdateService svcChild = table.getUpdateService(); + updateService = getSampleTypeUpdateService(childType.getName()); rows.clear(); rows.add(CaseInsensitiveHashMap.of("name", "C1", "MaterialInputs/Parent1Samples", "P1-1,P1-2", "MaterialInputs/Parent2Samples", "P2-1")); @@ -682,7 +749,7 @@ public void testUpdateSomeParents() throws Exception rows.add(CaseInsensitiveHashMap.of("name", "C4", "MaterialInputs/Parent1Samples", "P1-2", "MaterialInputs/Parent2Samples", "P2-2")); rows.add(CaseInsensitiveHashMap.of("name", "C5", "MaterialInputs/Parent1Samples", "P1-1, \"P1-4,test\", P1-2")); - inserted = svcChild.insertRows(user, c, rows, errors, null, null); + inserted = updateService.insertRows(user, c, rows, errors, null, null); assertFalse(errors.hasErrors()); assertEquals("Number of child samples inserted not as expected", 5, inserted.size()); @@ -691,7 +758,7 @@ public void testUpdateSomeParents() throws Exception rows.add(CaseInsensitiveHashMap.of("rowId", parent2Type.getSample(c, "P2-1").getRowId(), "MaterialInputs/ChildSamples", "C1")); try { - svcParent2.updateRows(user, c, rows, null, errors, null, null); + updateService.updateRows(user, c, rows, null, errors, null, null); fail("Expected to throw exception"); } catch (Exception e) @@ -700,14 +767,12 @@ public void testUpdateSomeParents() throws Exception } errors = new BatchValidationException(); - ExpMaterial P11 = parent1Type.getSample(c, "P1-1"); ExpMaterial P12 = parent1Type.getSample(c, "P1-2"); ExpMaterial P14 = parent1Type.getSample(c, "P1-4,test"); ExpMaterial P21 = parent2Type.getSample(c, "P2-1"); ExpMaterial P22 = parent2Type.getSample(c, "P2-2"); - ExpMaterial C1 = childType.getSample(c, "C1"); ExpMaterial C2 = childType.getSample(c, "C2"); ExpMaterial C4 = childType.getSample(c, "C4"); @@ -718,12 +783,29 @@ public void testUpdateSomeParents() throws Exception opts.setParents(true); opts.setDepth(2); + // Attempt to merge using rowIds + { + rows.clear(); + rows.add(CaseInsensitiveHashMap.of("rowId", C1.getRowId(), "name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); + rows.add(CaseInsensitiveHashMap.of("rowId", C4.getRowId(), "name", "C5", "MaterialInputs/Parent1Samples", null)); // intentionally mix up name + + try + { + updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); + fail("Expected to throw exception"); + } + catch (Exception e) + { + assertThat(e.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); + } + } + // now update the children with various types of modifications to the parentage rows.clear(); rows.add(CaseInsensitiveHashMap.of("name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); // change one parent but not the other rows.add(CaseInsensitiveHashMap.of("name", "C4", "MaterialInputs/Parent1Samples", null)); // remove one parent but not the other - svcChild.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); + updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); assertFalse(errors.hasErrors()); ExpLineage lineage = ExpLineageService.get().getLineage(c, user, C1, opts); @@ -735,12 +817,11 @@ public void testUpdateSomeParents() throws Exception assertFalse("Expected 'C4' to not be derived from 'P1-2'", lineage.getMaterials().contains(P12)); assertTrue("Expected 'C4' to still be derived from 'P2-2'", lineage.getMaterials().contains(P22)); - rows.clear(); rows.add(CaseInsensitiveHashMap.of("name", "C4", "MaterialInputs/Parent1Samples", "P1-1", "MaterialInputs/Parent2Samples", "P2-1")); // change both parents rows.add(CaseInsensitiveHashMap.of("name", "C2", "MaterialInputs/Parent1Samples", "", "MaterialInputs/Parent2Samples", null)); // remove both parents - svcChild.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); + updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); assertFalse(errors.hasErrors()); lineage = ExpLineageService.get().getLineage(c, user, C2, opts); @@ -774,9 +855,6 @@ public void testParentColAndDataInputDerivation() throws Exception final ExpSampleType st = createSampleType(sampleTypeName, props, null); // insert and derive with both 'parent' column and 'DataInputs/Samples' - UserSchema schema = QueryService.get().getUserSchema(user, c, SamplesSchema.SCHEMA_SAMPLES); - TableInfo table = schema.getTable(sampleTypeName); - QueryUpdateService svc = table.getUpdateService(); List> rows = new ArrayList<>(); rows.add(CaseInsensitiveHashMap.of("name", "A", "data", 10, "parent", null)); @@ -795,6 +873,7 @@ public void testParentColAndDataInputDerivation() throws Exception // F BatchValidationException errors = new BatchValidationException(); + QueryUpdateService svc = getSampleTypeUpdateService(st.getName()); List> inserted = svc.insertRows(user, c, rows, errors, null, null); assertFalse(errors.hasErrors()); assertEquals(6, inserted.size()); @@ -847,6 +926,7 @@ public void testParentColAndDataInputDerivation() throws Exception assertEquals("Expected 'E' and 'D' to be derived in the same run since they share 'B' and 'C' as parents", E.getRowId(), E.getRowId()); ExpRun derivationRun = E.getRun(); + assertNotNull(derivationRun); assertTrue(derivationRun.getMaterialInputs().containsKey(B)); assertTrue(derivationRun.getMaterialInputs().containsKey(C)); @@ -890,6 +970,7 @@ public void testParentColAndDataInputDerivation() throws Exception assertFalse(derivationRun2.getMaterialOutputs().contains(E)); ExpRun oldDerivationRun = ExperimentService.get().getExpRun(derivationRun.getRowId()); + assertNotNull(oldDerivationRun); assertEquals(oldDerivationRun.getRowId(), derivationRun.getRowId()); assertTrue(oldDerivationRun.getMaterialInputs().containsKey(B)); @@ -906,7 +987,7 @@ public void testParentColAndDataInputDerivation() throws Exception var multiValueColumn = "Outputs/Materials/" + sampleTypeName + "/Created"; var url = new ActionURL("query", "selectRows", c); - url.addParameter(QueryParam.schemaName, schema.getName()); + url.addParameter(QueryParam.schemaName, getSampleSchema().getName()); url.addParameter("query." + QueryParam.queryName, sampleTypeName); url.addParameter("query." + QueryParam.columns, "Name, " + multiValueColumn); url.addFilter("query", FieldKey.fromParts("Name"), CompareType.EQUAL, "A"); @@ -958,8 +1039,6 @@ public void testSampleTypeWithVocabularyProperties() throws Exception // create a sample type createSampleType(sampleName, List.of(new GWTPropertyDescriptor("name", "string")), null); - UserSchema schema = QueryService.get().getUserSchema(user, c, SchemaKey.fromParts("Samples")); - // insert a sample ArrayListMap row = new ArrayListMap<>(); row.put("name", "TestSample"); @@ -968,15 +1047,16 @@ public void testSampleTypeWithVocabularyProperties() throws Exception row.put(vocabularyPropertyURIs.get(helper.agePropertyName), null); // inserting a property with null value List> rows = helper.buildRows(row); - var insertedSample = helper.insertRows(c, rows ,sampleName, schema); + UserSchema schema = getSampleSchema(); + var insertedSample = helper.insertRows(c, rows, sampleName, schema); assertEquals("Custom Property is not inserted", sampleType, OntologyManager.getPropertyObjects(c, insertedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); - //Verifying property with null value is not inserted + // Verifying property with null value is not inserted assertEquals("Property with null value is present.", 0, OntologyManager.getPropertyObjects(c, vocabularyPropertyURIs.get(helper.agePropertyName)).size()); - //update inserted sample + // update inserted sample ArrayListMap rowToUpdate = new ArrayListMap<>(); rowToUpdate.put("name", "TestSample"); rowToUpdate.put("RowId", insertedSample.get(0).get("RowId")); @@ -995,10 +1075,10 @@ public void testSampleTypeWithVocabularyProperties() throws Exception assertEquals("Custom Property is not updated", updatedSampleType, OntologyManager.getPropertyObjects(c, updatedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); - //Verify property updated to a null value gets deleted + // Verify property updated to a null value gets deleted assertEquals("Property with null value is present.", 0, OntologyManager.getPropertyObjects(c, vocabularyPropertyURIs.get(helper.colorPropertyName)).size()); - //Verify property inserted during update rows in inserted + // Verify property inserted during update rows in inserted assertEquals("New Property is not inserted with update rows", sampleAge, OntologyManager.getPropertyObjects(c, updatedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.agePropertyName)).getFloatValue().intValue()); } @@ -1007,7 +1087,6 @@ public void testSampleTypeWithVocabularyProperties() throws Exception public void testDetailedAuditLog() throws Exception { User user = TestContext.get().getUser(); - UserSchema auditSchema = AuditLogService.get().createSchema(user, c); TableInfo auditTable = auditSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); Integer RowId = new SqlSelector(auditSchema.getDbSchema(), new SQLFragment("select max(rowid) FROM ").append(auditTable.getFromSQL("_"))) @@ -1020,27 +1099,17 @@ public void testDetailedAuditLog() throws Exception props.add(new GWTPropertyDescriptor("Value", "float")); final ExpSampleType st = createSampleType("SamplesDAL", props, null); - QuerySchema samplesSchema = DefaultSchema.get(user, c, "samples"); - assertNotNull(samplesSchema); - TableInfo samplesTable = samplesSchema.getTable("SamplesDAL"); - assertNotNull(samplesTable); - QueryUpdateService qus = samplesTable.getUpdateService(); - assertNotNull(qus); BatchValidationException errors = new BatchValidationException(); - Map config = Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, AuditBehaviorType.DETAILED); // insert a sample - List> rows = new ArrayList<>(); - rows.add(PageFlowUtil.mapInsensitive("Name", "A1", "Measure", "Initial", "Value", 1.0)); - List> ret = qus.insertRows(user, c, rows, errors, config, null); - String msg = !errors.getRowErrors().isEmpty() ? errors.getRowErrors().get(0).toString() : "no message"; - assertFalse(msg, errors.hasErrors()); - assertEquals(1,ret.size()); + List> ret = insertSampleRows(st.getName(), List.of(CaseInsensitiveHashMap.of("Name", "A1", "Measure", "Initial", "Value", 1.0))); + assertEquals(1, ret.size()); assertNotNull(ret.get(0).get("rowid")); int rowid = (int) JdbcType.INTEGER.convert(ret.get(0).get("rowid")); + // check audit log - SimpleFilter f = new SimpleFilter(new FieldKey(null,"RowId"),auditMaxRowid, CompareType.GT); - List events = AuditLogService.get().getAuditEvents(c,user,SampleTimelineAuditEvent.EVENT_TYPE,f,new Sort("-RowId")); + SimpleFilter f = new SimpleFilter(new FieldKey(null, "RowId"), auditMaxRowid, CompareType.GT); + List events = AuditLogService.get().getAuditEvents(c, user, SampleTimelineAuditEvent.EVENT_TYPE, f, new Sort("-RowId")); assertFalse(events.isEmpty()); assertNull(events.get(0).getOldRecordMap()); assertNotNull(events.get(0).getNewRecordMap()); @@ -1054,12 +1123,10 @@ public void testDetailedAuditLog() throws Exception assertNull(newRecordMap.get("AliquotUnit")); // UPDATE - rows.clear(); errors.clear(); - rows.add(PageFlowUtil.mapInsensitive("RowId", rowid, "Measure", "Updated", "Value", 2.0)); - qus.updateRows(user, c, rows, null, errors, config, null); - assertFalse(errors.hasErrors()); + updateSampleRows(st.getName(), List.of(CaseInsensitiveHashMap.of("RowId", rowid, "Measure", "Updated", "Value", 2.0))); + // check audit log - events = AuditLogService.get().getAuditEvents(c,user,SampleTimelineAuditEvent.EVENT_TYPE,f,new Sort("-RowId")); + events = AuditLogService.get().getAuditEvents(c, user, SampleTimelineAuditEvent.EVENT_TYPE, f, new Sort("-RowId")); assertFalse(events.isEmpty()); assertNotNull(events.get(0).getOldRecordMap()); Map oldRecordMap = new CaseInsensitiveHashMap<>(PageFlowUtil.mapFromQueryString(events.get(0).getOldRecordMap())); @@ -1076,12 +1143,11 @@ public void testDetailedAuditLog() throws Exception // MERGE // and since merge is a different code path... - rows.clear(); errors.clear(); - rows.add(PageFlowUtil.mapInsensitive("Name", "A1", "Measure", "Merged", "Value", 3.0)); - int count = qus.mergeRows(user, c, MapDataIterator.of(rows), errors, config, null); + int count = mergeSampleRows(st.getName(), List.of(CaseInsensitiveHashMap.of("Name", "A1", "Measure", "Merged", "Value", 3.0))); assertEquals(1, count); + // check audit log - events = AuditLogService.get().getAuditEvents(c,user,SampleTimelineAuditEvent.EVENT_TYPE,f,new Sort("-RowId")); + events = AuditLogService.get().getAuditEvents(c, user, SampleTimelineAuditEvent.EVENT_TYPE, f, new Sort("-RowId")); assertFalse(events.isEmpty()); assertNotNull(events.get(0).getOldRecordMap()); oldRecordMap = new CaseInsensitiveHashMap<>(PageFlowUtil.mapFromQueryString(events.get(0).getOldRecordMap())); @@ -1105,26 +1171,19 @@ public void testDetailedAuditLog() throws Exception @Test public void testExpMaterialPermissions() throws Exception { - User user = TestContext.get().getUser(); - var schema = QueryService.get().getUserSchema(user, c, ExpSchema.SCHEMA_EXP); - // create a sample type ExpSampleType st = createSampleType("MySamples", List.of(new GWTPropertyDescriptor("name", "string")), null); // insert a sample - var errors = new BatchValidationException(); - List> ret = QueryService.get().getUserSchema(user, c, SamplesSchema.SCHEMA_SAMPLES) - .getTable("MySamples") - .getUpdateService() - .insertRows(user, c, List.of(CaseInsensitiveHashMap.of("name", "SampleInSampleType")), errors, null, null); - String msg = !errors.getRowErrors().isEmpty() ? errors.getRowErrors().get(0).toString() : "no message"; - assertFalse(msg, errors.hasErrors()); - assertEquals(1,ret.size()); + List> ret = insertSampleRows(st.getName(), List.of(CaseInsensitiveHashMap.of("name", "SampleInSampleType"))); + assertEquals(1, ret.size()); assertNotNull(ret.get(0).get("rowid")); long stSampleId = (long) JdbcType.BIGINT.convert(ret.get(0).get("rowid")); // verify insert, update aren't allowed, but read and delete are allowed - var materialsTable = schema.getTable(ExpSchema.TableType.Materials.name()); + User user = TestContext.get().getUser(); + var schema = QueryService.get().getUserSchema(user, c, ExpSchema.SCHEMA_EXP); + var materialsTable = schema.getTableOrThrow(ExpSchema.TableType.Materials.name()); var qus = materialsTable.getUpdateService(); assertNotNull(qus); assertTrue(qus.hasPermission(user, ReadPermission.class)); @@ -1132,7 +1191,7 @@ public void testExpMaterialPermissions() throws Exception assertFalse(qus.hasPermission(user, InsertPermission.class)); assertFalse(qus.hasPermission(user, UpdatePermission.class)); - // create a sample outside of a SampleType + // create a sample outside a SampleType var lsid = ExperimentService.get().generateLSID(c, ExpMaterial.class, "SampleNotInSampleType"); ExpMaterial m = ExperimentService.get().createExpMaterial(c, Lsid.parse(lsid)); m.save(user); @@ -1191,10 +1250,8 @@ public void testInsertOptionUpdate() throws Exception final String sampleTypeName = "TestSamplesWithRequired"; ExpSampleType sampleType = createSampleType(sampleTypeName, props, null); - TableInfo table = getSampleTypeTable(sampleTypeName); - QueryUpdateService qus = table.getUpdateService(); - assertNotNull(qus); + QueryUpdateService qus = getSampleTypeUpdateService(sampleTypeName); String longFieldAlias = table.getColumn(longFieldName).getAlias().getId(); assertFalse("Unexpected long field alias", longFieldName.equalsIgnoreCase(longFieldAlias)); @@ -1212,7 +1269,7 @@ public void testInsertOptionUpdate() throws Exception assertFalse(context.getErrors().hasErrors()); assertEquals("Unexpected count from IMPORT on loadRows()", 3, count); - // Issue 53168: aliquot cannot be parent to its aliquot parent + // Issue 53168: aliquot cannot be a parent to its aliquot parent BatchValidationException errors = new BatchValidationException(); List> rows = new ArrayList<>(); rows.add(CaseInsensitiveHashMap.of("rowId", sampleType.getSample(c, "S-1").getRowId(), "MaterialInputs/" + sampleTypeName, "S-1-1")); @@ -1334,10 +1391,16 @@ private ExpSampleType createSampleType(String sampleTypeName, List> insertSampleRows(String sampleType, List> rows) throws Exception +{ + BatchValidationException errors = new BatchValidationException(); + QueryUpdateService svc = getSampleTypeUpdateService(sampleType); + Map config = Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, AuditBehaviorType.DETAILED); + int count = svc.mergeRows(TestContext.get().getUser(), c, MapDataIterator.of(rows), errors, config, null); + if (errors.hasErrors()) + throw errors; + return count; +} + +private List> updateSampleRows(String sampleType, List> rows) throws Exception +{ + BatchValidationException errors = new BatchValidationException(); + QueryUpdateService svc = getSampleTypeUpdateService(sampleType); + Map config = Map.of(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, AuditBehaviorType.DETAILED); + List> ret = svc.updateRows(TestContext.get().getUser(), c, rows, null, errors, config, null); + if (errors.hasErrors()) + throw errors; + return ret; +} %> diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 57f41d3cce7..5c84a37418a 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -244,7 +244,7 @@ public void configureDataIteratorContext(DataIteratorContext context) protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) { assert _sampleType != null : "SampleType required for insert/update, but not required for read/delete"; - return new PrepareDataIteratorBuilder(_sampleType, (ExpMaterialTableImpl) getQueryTable(), in, getContainer(), getUser()); + return new PreTriggerDataIteratorBuilder(_sampleType, (ExpMaterialTableImpl) getQueryTable(), in, getContainer(), getUser()); } @Override @@ -1631,7 +1631,7 @@ void audit(QueryService.AuditAction auditAction) } // TODO: validate/compare functionality of CoerceDataIterator and loadRows() - private static class PrepareDataIteratorBuilder implements DataIteratorBuilder + private static class PreTriggerDataIteratorBuilder implements DataIteratorBuilder { private static final int BATCH_SIZE = 100; @@ -1641,7 +1641,7 @@ private static class PrepareDataIteratorBuilder implements DataIteratorBuilder final Container container; final User user; - public PrepareDataIteratorBuilder(@NotNull ExpSampleTypeImpl sampleType, ExpMaterialTableImpl materialTable, DataIteratorBuilder in, Container container, User user) + public PreTriggerDataIteratorBuilder(@NotNull ExpSampleTypeImpl sampleType, ExpMaterialTableImpl materialTable, DataIteratorBuilder in, Container container, User user) { this.sampleType = sampleType; this.builder = in; @@ -1654,10 +1654,11 @@ public PrepareDataIteratorBuilder(@NotNull ExpSampleTypeImpl sampleType, ExpMate public DataIterator getDataIterator(DataIteratorContext context) { DataIterator source = LoggingDataIterator.wrap(builder.getDataIterator(context)); + boolean isMerge = context.getInsertOption() == InsertOption.MERGE; boolean isUpdate = context.getInsertOption() == InsertOption.UPDATE; // drop columns - ColumnInfo containerColumn = this.materialTable.getColumn(this.materialTable.getContainerFieldKey()); + ColumnInfo containerColumn = materialTable.getColumn(materialTable.getContainerFieldKey()); String containerFieldLabel = containerColumn.getLabel(); var drop = new CaseInsensitiveHashSet(); for (int i = 1; i <= source.getColumnCount(); i++) @@ -1690,8 +1691,13 @@ public DataIterator getDataIterator(DataIteratorContext context) continue; if (isContainerField && context.isCrossFolderImport() && !context.getInsertOption().updateOnly) continue; - if (isUpdate && RowId.name().equalsIgnoreCase(name)) - continue; + if (RowId.name().equalsIgnoreCase(name)) + { + if (isUpdate) + continue; + if (isMerge) + throw new IllegalArgumentException("RowId is not accepted when merging samples. Specify only the sample name instead."); + } drop.add(name); } } From ac7dd25477bc0ab80185309f70bb8dd3a4cc1cca Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Sun, 7 Dec 2025 23:59:07 -0800 Subject: [PATCH 38/62] Handle "Row Id" labeling --- .../SampleUpdateAddColumnsDataIterator.java | 3 +- .../experiment/api/ExpSampleTypeTestCase.jsp | 21 +++++- .../api/SampleTypeUpdateServiceDI.java | 72 ++++++++++--------- 3 files changed, 58 insertions(+), 38 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java b/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java index e4bf7230f05..68a44b09d91 100644 --- a/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/SampleUpdateAddColumnsDataIterator.java @@ -40,7 +40,7 @@ public class SampleUpdateAddColumnsDataIterator extends WrapperDataIterator final IntHashMap aliquotRoots = new IntHashMap<>(); final IntHashMap sampleState = new IntHashMap<>(); - public SampleUpdateAddColumnsDataIterator(DataIterator in, TableInfo target, long sampleTypeId, boolean useRowId) + public SampleUpdateAddColumnsDataIterator(DataIterator in, TableInfo target, long sampleTypeId, String keyColumnName) { super(in); this._unwrapped = (CachingDataIterator)in; @@ -52,7 +52,6 @@ public SampleUpdateAddColumnsDataIterator(DataIterator in, TableInfo target, lon this._rootMaterialRowIdColIndex = map.get(RootMaterialRowId.name()); this._currentSampleStateColIndex = map.get(CURRENT_SAMPLE_STATUS_COLUMN_NAME); - String keyColumnName = useRowId ? RowId.name() : Name.name(); Integer index = map.get(keyColumnName); ColumnInfo col = target.getColumn(keyColumnName); if (null == index || null == col) diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index c2c85b29a5a..7dff4129596 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -550,8 +550,8 @@ public void testAliases() throws Exception // Update, keyed by rowId { rows = new ArrayList<>(); - rows.add(CaseInsensitiveHashMap.of("rowId", fooSample.getRowId(), "alias", "ken, griffey", "flag", "norway")); - rows.add(CaseInsensitiveHashMap.of("rowId", booSample.getRowId(), "alias", "edgar, martinez", "flag", "sweden")); + rows.add(CaseInsensitiveHashMap.of("Row Id", fooSample.getRowId(), "alias", "ken, griffey", "flag", "norway")); + rows.add(CaseInsensitiveHashMap.of("Row Id", booSample.getRowId(), "alias", "edgar, martinez", "flag", "sweden")); updateSampleRows(st.getName(), rows); @@ -800,6 +800,23 @@ public void testUpdateSomeParents() throws Exception } } + // Attempt to merge using "Row Id" label + { + rows.clear(); + rows.add(CaseInsensitiveHashMap.of("Row Id", C1.getRowId(), "name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); + rows.add(CaseInsensitiveHashMap.of("Row Id", C4.getRowId(), "name", "C5", "MaterialInputs/Parent1Samples", null)); // intentionally mix up name + + try + { + updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); + fail("Expected to throw exception"); + } + catch (Exception e) + { + assertThat(e.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); + } + } + // now update the children with various types of modifications to the parentage rows.clear(); rows.add(CaseInsensitiveHashMap.of("name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); // change one parent but not the other diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 5c84a37418a..662480cc82a 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -137,6 +137,7 @@ import static org.labkey.api.audit.AuditHandler.DELTA_PROVIDED_DATA_PREFIX; import static org.labkey.api.audit.AuditHandler.PROVIDED_DATA_PREFIX; import static org.labkey.api.data.TableSelector.ALL_COLUMNS; +import static org.labkey.api.dataiterator.DataIteratorUtil.DUPLICATE_COLUMN_IN_DATA_ERROR; import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs; import static org.labkey.api.dataiterator.SampleUpdateAddColumnsDataIterator.CURRENT_SAMPLE_STATUS_COLUMN_NAME; import static org.labkey.api.exp.api.ExpRunItem.PARENT_IMPORT_ALIAS_MAP_PROP; @@ -711,7 +712,7 @@ public Map moveRows(User user, Container container, Container ta private Map> getMaterialsForMoveRows(Container container, Container targetContainer, List> rows, BatchValidationException errors) { - Set sampleIds = rows.stream().map(row -> MapUtils.getLong(row,RowId.name())).collect(Collectors.toSet()); + Set sampleIds = rows.stream().map(row -> MapUtils.getLong(row, RowId.name())).collect(Collectors.toSet()); if (sampleIds.isEmpty()) { errors.addRowError(new ValidationException("Sample IDs must be specified for the move operation.")); @@ -1681,17 +1682,17 @@ public DataIterator getDataIterator(DataIteratorContext context) continue; if (isAliasHeader(name)) continue; - if (isSampleStateHeader(name)) + if (isExpMaterialColumn(SampleState, name)) continue; - if (isMaterialExpDateHeader(name)) + if (isExpMaterialColumn(MaterialExpDate, name)) continue; - if (isStoredAmountHeader(name)) + if (isExpMaterialColumn(StoredAmount, name)) continue; - if (isUnitsHeader(name)) + if (isExpMaterialColumn(Units, name)) continue; if (isContainerField && context.isCrossFolderImport() && !context.getInsertOption().updateOnly) continue; - if (RowId.name().equalsIgnoreCase(name)) + if (isExpMaterialColumn(RowId, name)) { if (isUpdate) continue; @@ -1723,11 +1724,17 @@ public DataIterator getDataIterator(DataIteratorContext context) addAliquotedFrom.addNullColumn(PARENT_RECOMPUTE_NAME_COL, JdbcType.VARCHAR); addAliquotedFrom.selectAll(); - var addRequiredColsDI = new SampleUpdateAddColumnsDataIterator(new CachingDataIterator(addAliquotedFrom), materialTable, sampleType.getRowId(), columnNameMap.containsKey(RowId.name())); + DataIterator di = new SampleUpdateAddColumnsDataIterator( + new CachingDataIterator(addAliquotedFrom), + materialTable, + sampleType.getRowId(), + getKeyColumnAlias(materialTable, columnNameMap) + ); - SimpleTranslator c = new _SamplesCoerceDataIterator(addRequiredColsDI, context, sampleType, materialTable); + di = new _SamplesCoerceDataIterator(di, context, sampleType, materialTable); context.setWithLookupRemapping(false); - return LoggingDataIterator.wrap(c); + + return LoggingDataIterator.wrap(di); } // CoerceDataIterator to handle the lookup/alternatekeys functionality of loadRows(), @@ -1752,19 +1759,36 @@ public DataIterator getDataIterator(DataIteratorContext context) addColumns.addNullColumn(ROOT_RECOMPUTE_ROWID_COL, JdbcType.INTEGER); addColumns.addNullColumn(PARENT_RECOMPUTE_NAME_COL, JdbcType.VARCHAR); } - DataIterator dataIterator = LoggingDataIterator.wrap(addColumns); + + DataIterator di = LoggingDataIterator.wrap(addColumns); // Table Counters - DataIteratorBuilder dib = ExpDataIterators.CounterDataIteratorBuilder.create(dataIterator, sampleType.getContainer(), materialTable, ExpSampleType.SEQUENCE_PREFIX, sampleType.getRowId()); - dataIterator = dib.getDataIterator(context); + di = ExpDataIterators.CounterDataIteratorBuilder + .create(di, sampleType.getContainer(), materialTable, ExpSampleType.SEQUENCE_PREFIX, sampleType.getRowId()) + .getDataIterator(context); // sampleset.createSampleNames() + generate lsid // TODO: does not handle insertIgnore - DataIterator names = new _GenerateNamesDataIterator(sampleType, container, user, DataIteratorUtil.wrapMap(dataIterator, false), context, batchSize); - + DataIterator names = new _GenerateNamesDataIterator(sampleType, container, user, DataIteratorUtil.wrapMap(di, false), context, batchSize); return LoggingDataIterator.wrap(names); } + private static @NotNull String getKeyColumnAlias(TableInfo materialTable, @NotNull Map columnNameMap) + { + // Currently, SampleUpdateAddColumnsDataIterator is being called before a translator is invoked to + // remap column labels to columns (e.g., "Row Id" -> "RowId"). Due to this, we need to search the + // map of columns for the key column. + var rowIdAliases = ImportAliasable.Helper.createImportSet(materialTable.getColumn(RowId.fieldKey())); + rowIdAliases.retainAll(columnNameMap.keySet()); + + if (rowIdAliases.size() == 1) + return rowIdAliases.iterator().next(); + if (rowIdAliases.isEmpty()) + return Name.name(); + + throw new IllegalArgumentException(String.format(DUPLICATE_COLUMN_IN_DATA_ERROR, RowId.name())); + } + private static boolean isReservedHeader(String name) { if (isNameHeader(name) || isDescriptionHeader(name) || isCommentHeader(name) || "CpasType".equalsIgnoreCase(name) || isAliasHeader(name)) @@ -1794,11 +1818,6 @@ private static boolean isDescriptionHeader(String name) return isExpMaterialColumn(Description, name); } - private static boolean isSampleStateHeader(String name) - { - return isExpMaterialColumn(SampleState, name); - } - private static boolean isCommentHeader(String name) { return isExpMaterialColumn(Flag, name) || "Comment".equalsIgnoreCase(name); @@ -1809,21 +1828,6 @@ private static boolean isAliasHeader(String name) return isExpMaterialColumn(Alias, name); } - private static boolean isMaterialExpDateHeader(String name) - { - return isExpMaterialColumn(MaterialExpDate, name); - } - - private static boolean isStoredAmountHeader(String name) - { - return isExpMaterialColumn(StoredAmount, name) || StoredAmount.label().equalsIgnoreCase(name); - } - - public static boolean isUnitsHeader(String name) - { - return isExpMaterialColumn(Units, name); - } - private static boolean isAliquotRollupHeader(String name) { Set rollupFields = new CaseInsensitiveHashSet(); From 1418297fc196071dd061d67aad74ce2f7de28e1b Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Sun, 7 Dec 2025 12:26:33 -0800 Subject: [PATCH 39/62] Sample: remove row-by-row update - data iterator is the only update pathway now --- .../test/integration/SampleTypeCrud.ispec.ts | 3 +- .../experiment/api/ExpSampleTypeTestCase.jsp | 1 + .../api/SampleTypeUpdateServiceDI.java | 341 ++---------------- 3 files changed, 39 insertions(+), 306 deletions(-) diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index 656632f1864..32b65e02220 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -1084,8 +1084,7 @@ describe('Amount/Unit CRUD', () => { }] }, { ...topFolderOptions, ...editorUserOptions }).expect((result) => { const errorResp = JSON.parse(result.text); - // Note that the row by row update error is different from DIB. This is OK for now since we are planning to deprecate row by row updates. - expect(errorResp['exception']).toContain("Value '-1000.0 (g)' for field 'Amount' is invalid. Amounts must be non-negative."); + expect(errorResp['exception']).toContain("Value '-1' for field 'Amount' is invalid. Amounts must be non-negative."); }); // Using data iterator diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index 7dff4129596..d2f5fa15aac 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -1088,6 +1088,7 @@ public void testSampleTypeWithVocabularyProperties() throws Exception oldKey.put("RowId", insertedSample.get(0).get("RowId")); oldKeys.add(oldKey); + // TODO: Figure out why LSID is no longer being pulled through on the row (even though this was previously invoking DIB pathway) var updatedSample = helper.updateRows(c, rowsToUpdate, oldKeys, sampleName, schema); assertEquals("Custom Property is not updated", updatedSampleType, OntologyManager.getPropertyObjects(c, updatedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 662480cc82a..40f2727d51b 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -16,16 +16,12 @@ package org.labkey.experiment.api; import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.converters.IntegerConverter; -import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Strings; -import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.labkey.api.assay.AssayFileWriter; import org.labkey.api.audit.AuditLogService; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; @@ -50,7 +46,6 @@ import org.labkey.api.data.RemapCache; import org.labkey.api.data.RuntimeSQLException; import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; import org.labkey.api.data.UpdateableTableInfo; @@ -100,7 +95,6 @@ import org.labkey.api.query.ValidationException; import org.labkey.api.reader.ColumnDescriptor; import org.labkey.api.reader.DataLoader; -import org.labkey.api.search.SearchService; import org.labkey.api.security.User; import org.labkey.api.security.permissions.MoveEntitiesPermission; import org.labkey.api.security.permissions.ReadPermission; @@ -116,7 +110,6 @@ import org.labkey.experiment.SampleTypeAuditProvider; import java.io.IOException; -import java.nio.file.Path; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; @@ -143,14 +136,12 @@ import static org.labkey.api.exp.api.ExpRunItem.PARENT_IMPORT_ALIAS_MAP_PROP; import static org.labkey.api.exp.api.ExperimentService.QueryOptions.SkipBulkRemapCache; import static org.labkey.api.exp.api.SampleTypeDomainKind.ALIQUOT_ROLLUP_FIELD_LABELS; -import static org.labkey.api.exp.api.SampleTypeDomainKind.SAMPLE_TYPE_FILE_DIRECTORY_NAME; import static org.labkey.api.exp.api.SampleTypeService.ConfigParameters.SkipAliquotRollup; import static org.labkey.api.exp.api.SampleTypeService.ConfigParameters.SkipMaxSampleCounterFunction; import static org.labkey.api.exp.api.SampleTypeService.MISSING_AMOUNT_ERROR_MESSAGE; import static org.labkey.api.exp.api.SampleTypeService.MISSING_UNITS_ERROR_MESSAGE; import static org.labkey.api.exp.api.SampleTypeService.UNPROVIDED_VALUE_ERROR_MESSAGE_PATTERN; import static org.labkey.api.exp.query.ExpMaterialTable.Column.*; -import static org.labkey.api.exp.query.SamplesSchema.SCHEMA_SAMPLES; import static org.labkey.api.util.IntegerUtils.asLong; import static org.labkey.experiment.ExpDataIterators.incrementCounts; import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.insert; @@ -158,13 +149,7 @@ import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.update; /** - * - * This replaces the old row at a time UploadSamplesHelper.uploadMaterials() implementations. - * - * originally copied from ExpDataClassDataTableImpl.DataClassDataUpdateService, - * - * TODO find remaining shared code and refactor - * + * QueryUpdateService implementation for samples in sample types. */ public class SampleTypeUpdateServiceDI extends DefaultQueryUpdateService { @@ -523,17 +508,6 @@ public static void confirmAmountAndUnitsColumns(Collection columns) throw new ConversionExceptionWithMessage(MISSING_UNITS_ERROR_MESSAGE); } - private static boolean useDataIteratorForUpdate( - List> rows, - List> oldKeys - ) - { - if (rows == null || rows.isEmpty() || oldKeys != null) - return false; - - return hasUniformKeys(rows); - } - @Override public List> updateRows( User user, @@ -546,65 +520,49 @@ public List> updateRows( ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException { assert _sampleType != null : "SampleType required for insert/update, but not required for read/delete"; - if (rows != null && !rows.isEmpty()) - confirmAmountAndUnitsColumns(rows.get(0).keySet()); - boolean useDib = useDataIteratorForUpdate(rows, oldKeys); + if (rows == null || rows.isEmpty()) + return Collections.emptyList(); List> results; DbScope scope = getSchema().getDbSchema().getScope(); - if (useDib) - { - Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; - recordDataIteratorUsed(configParameters); + Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; + recordDataIteratorUsed(configParameters); - try - { - results = scope.executeWithRetry(transaction -> - { - var context = getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters); - var ret = super._updateRowsUsingDIB(user, container, rows, context, extraScriptContext); - // we need to throw if we don't want executeWithRetry() attempt commit() - if (context.getErrors().hasErrors()) - throw new DbScope.RetryPassthroughException(context.getErrors()); - return ret; - }); - } - catch (DbScope.RetryPassthroughException retryException) - { - retryException.rethrow(BatchValidationException.class); - throw retryException.throwRuntimeException(); - } - } - else + try { - results = super.updateRows(user, container, rows, oldKeys, errors, configParameters, extraScriptContext); - - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified); - scope.addCommitTask(() -> + results = scope.executeWithRetry(transaction -> { - List orderedRowIds = new ArrayList<>(); - for (Map result : results) + var context = getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters); + int index = 0; + List> ret = new ArrayList<>(); + + while (index < rows.size()) { - Long rowId = MapUtils.getLong(result, RowId.name()); - if (rowId != null) - orderedRowIds.add(rowId); - } - // Issue 51263: order by RowId to reduce deadlock - Collections.sort(orderedRowIds); + var rowKeys = new CaseInsensitiveHashSet(rows.get(index).keySet()); + confirmAmountAndUnitsColumns(rowKeys); - ExpMaterialTableImpl tableInfo = (ExpMaterialTableImpl) QueryService.get().getUserSchema(User.getSearchUser(), container, SCHEMA_SAMPLES).getTable(_sampleType.getName()); - ListUtils.partition(orderedRowIds, 100).forEach(sublist -> - queue.addRunnable((q) -> - { - for (ExpMaterialImpl expMaterial : ExperimentServiceImpl.get().getExpMaterials(sublist)) - expMaterial.index(q, tableInfo); - }) - ); - }, DbScope.CommitTaskOption.POSTCOMMIT); + var nextIndex = index + 1; + while (nextIndex < rows.size() && rowKeys.equals(new CaseInsensitiveHashSet(rows.get(nextIndex).keySet()))) + nextIndex++; + + var rowsToProcess = rows.subList(index, nextIndex); + index = nextIndex; + + var subRet = super._updateRowsUsingDIB(user, container, rowsToProcess, context, extraScriptContext); + if (subRet != null) + ret.addAll(subRet); + } - /* setup mini dataiterator pipeline to process lineage */ - DataIterator di = _toDataIteratorBuilder("updateRows.lineage", results).getDataIterator(new DataIteratorContext()); - ExpDataIterators.derive(user, container, di, true, _sampleType, true); + // we need to throw if we don't want executeWithRetry() attempt commit() + if (context.getErrors().hasErrors()) + throw new DbScope.RetryPassthroughException(context.getErrors()); + return ret; + }); + } + catch (DbScope.RetryPassthroughException retryException) + { + retryException.rethrow(BatchValidationException.class); + throw retryException.throwRuntimeException(); } if (results != null && !results.isEmpty() && !errors.hasErrors()) @@ -777,17 +735,6 @@ protected Map _select(Container container, Object[] keys) throws throw new IllegalStateException("Overridden .getRow()/.getRows() calls .getMaterialMap()"); } - public Set getAliquotSpecificFields() - { - Domain domain = getDomain(); - Set fields = domain.getProperties().stream() - .filter(dp -> ExpSchema.DerivationDataScopeType.ChildOnly.name().equalsIgnoreCase(dp.getDerivationDataScope())) - .map(ImportAliasable::getName) - .collect(Collectors.toSet()); - - return new CaseInsensitiveHashSet(fields); - } - public Set getSampleMetaFields() { Domain domain = getDomain(); @@ -831,29 +778,6 @@ public static boolean isAliquotStatusChangeNeedRecalc(Collection available return false; } - // Customize negative amount error message when the provided unit doesn't match sample type unit. - // For example, provided value of "-1 kg" would have been converted to "-1000 mg" by now. - // This updateRow (going to be deprecated) inconsistent with the data iterator code path, which use provided value "-1" in error message. - // TODO: remove this override when consolidating sample update method to remove row by row update - @Override - protected void validateUpdateRow(Map row) throws ValidationException - { - for (ColumnInfo col : getQueryTable().getColumns()) - { - if (row.containsKey(col.getColumnName())) - { - // if provided value is present, validate provided - Object value = row.get(col.getColumnName()); - Object providedValue = null; - if (_sampleType != null && _sampleType.getMetricUnit() != null && value != null && (StoredAmount.name().equalsIgnoreCase(col.getColumnName()) || "Amount".equalsIgnoreCase(col.getColumnName()))) - { - providedValue = value + " (" + _sampleType.getMetricUnit() + ")"; - } - validateValue(col, value, providedValue); - } - } - } - @Override protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, boolean allowOwner, boolean retainCreation) throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException @@ -882,186 +806,9 @@ protected Map updateRow(User user, Container container, Map _update(User user, Container c, Map row, Map oldRow, Object[] keys) throws SQLException, ValidationException + protected Map _update(User user, Container c, Map row, Map oldRow, Object[] keys) { - assert _sampleType != null : "SampleType required for insert/update, but not required for read/delete"; - // LSID was stripped by super.updateRows() and is needed to insert into the dataclass provisioned table - String lsid = (String) oldRow.get("lsid"); - if (lsid == null) - throw new ValidationException("lsid required to update row"); - - Long rowId = asLong(oldRow.get(RowId.name())); - if (rowId == null) - throw new ValidationException(RowId.name() + " required to update row"); - - /** See {@link ExpDataIterators.SampleUpdateOnlyDataIteratorBuilder} for data iterator logical equivalent */ - String newName = (String) row.get(Name.name()); - if (row.containsKey(Name.name()) && StringUtils.isEmpty(newName)) - throw new ValidationException("Sample name cannot be blank"); - - /** See {@link ExpDataIterators.SampleNameChangeDataIterator} for data iterator logical equivalent */ - String oldName = (String) oldRow.get(Name.name()); - boolean hasNameChange = !StringUtils.isEmpty(newName) && !newName.equals(oldName); - if (hasNameChange && !NameExpressionOptionService.get().getAllowUserSpecificNamesValue(c)) - throw new ValidationException("User-specified sample name not allowed"); - - String oldAliquotedFromLSID = (String) oldRow.get(AliquotedFromLSID.name()); - boolean isAliquot = !StringUtils.isEmpty(oldAliquotedFromLSID); - - /** See {@link ExpDataIterators.AliquotRollupDataIterator} for data iterator logical equivalent */ - Integer aliquotRollupRoot = null; - SampleTypeService stService = SampleTypeService.get(); - if (!_sampleType.isMedia() && isAliquot) - { - Integer aliquotRoot = (Integer) oldRow.get(RootMaterialRowId.name()); - - if (row.containsKey(StoredAmount.name()) || row.containsKey(Units.name())) - { - Unit oldRowUnits = stService.getValidatedUnit(oldRow.get(Units.name()), _sampleType.getBaseUnit(), _sampleType.getName()); - Unit rowUnits = stService.getValidatedUnit(row.get(Units.name()), _sampleType.getBaseUnit(), _sampleType.getName()); - Quantity oldQuantity = null; - Quantity newQuantity = null; - if (oldRowUnits != null && oldRow.get(StoredAmount.name()) != null) - oldQuantity = Quantity.of((Number) oldRow.get(StoredAmount.name()), oldRowUnits); - if (rowUnits != null && row.get(StoredAmount.name()) != null) - newQuantity = Quantity.of((Number) row.get(StoredAmount.name()), rowUnits); - - if (newQuantity != null && (oldQuantity == null || !oldQuantity.equals(newQuantity))) - { - if (aliquotRoot != null) - aliquotRollupRoot = aliquotRoot; - } - } - - if (aliquotRollupRoot == null && row.containsKey(SampleState.name())) - { - List availableSampleStatuses = new ArrayList<>(); - if (SampleStatusService.get().supportsSampleStatus()) - { - for (DataState state: SampleStatusService.get().getAllProjectStates(c)) - { - if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) - availableSampleStatuses.add(state.getRowId()); - } - } - - if (!availableSampleStatuses.isEmpty()) - { - Long oldState = asLong(oldRow.get(SampleState.name())); - Long newState = asLong(row.get(SampleState.name())); - if (isAliquotStatusChangeNeedRecalc(availableSampleStatuses, oldState, newState)) - aliquotRollupRoot = aliquotRoot; - } - } - } - - Set aliquotFields = getAliquotSpecificFields(); - Set sampleMetaFields = getSampleMetaFields(); - - // Replace attachment columns with filename and keep AttachmentFiles - Map rowCopy = new CaseInsensitiveHashMap<>(); - - // remove aliquotedFrom from row, or error out - rowCopy.putAll(row); - String newAliquotedFromLSID = (String) rowCopy.get(AliquotedFromLSID.name()); - if (!StringUtils.isEmpty(newAliquotedFromLSID) && !newAliquotedFromLSID.equals(oldAliquotedFromLSID)) - throw new ValidationException("Updating aliquotedFrom is not supported"); - rowCopy.remove(AliquotedFromLSID.name()); - rowCopy.remove(RootMaterialRowId.name()); - rowCopy.remove(ExpMaterial.ALIQUOTED_FROM_INPUT); - - /** See {@link ExpDataIterators.SampleStatusCheckDataIterator} for data iterator logical equivalent */ - // We need to allow updating from one locked status to another locked status, but without other changes - // and updating from either locked or unlocked to something else while also updating other metadata - DataState oldStatus = SampleStatusService.get().getStateForRowId(getContainer(), MapUtils.getLong(oldRow,SampleState.name())); - boolean oldAllowsOp = SampleStatusService.get().isOperationPermitted(oldStatus, SampleTypeService.SampleOperations.EditMetadata); - DataState newStatus = SampleStatusService.get().getStateForRowId(getContainer(), MapUtils.getLong(rowCopy,SampleState.name())); - boolean newAllowsOp = SampleStatusService.get().isOperationPermitted(newStatus, SampleTypeService.SampleOperations.EditMetadata); - - Map ret = new CaseInsensitiveHashMap<>(super._update(user, c, rowCopy, oldRow, keys)); - - if (aliquotRollupRoot != null) - ret.put(ROOT_RECOMPUTE_ROWID_COL, aliquotRollupRoot); - - Map validRowCopy = new CaseInsensitiveHashMap<>(); - boolean hasNonStatusChange = false; - boolean hasStatusCol = false; - for (String updateField : rowCopy.keySet()) - { - Object updateValue = rowCopy.get(updateField); - boolean isAliquotField = aliquotFields.contains(updateField); - boolean isSampleMetaField = sampleMetaFields.contains(updateField); - - if (isAliquot && isSampleMetaField) - { - Object oldMetaValue = oldRow.get(updateField); - if (!Objects.equals(oldMetaValue, updateValue)) - LOG.warn("Sample metadata update has been skipped for an aliquot"); - } - else if (!isAliquot && isAliquotField) - { - LOG.warn("Aliquot-specific field update has been skipped for a sample."); - } - else - { - hasNonStatusChange = hasNonStatusChange || !SampleTypeServiceImpl.statusUpdateColumns.contains(updateField.toLowerCase()); - validRowCopy.put(updateField, updateValue); - } - - if (SampleState.name().equalsIgnoreCase(updateField)) - hasStatusCol = true; - } - // had a locked status before and either not updating the status or updating to a new locked status - if (hasNonStatusChange && !oldAllowsOp && (!hasStatusCol || !newAllowsOp)) - { - throw new ValidationException(String.format("Updating sample data when status is %s is not allowed.", oldStatus.getLabel())); - } - - /** See {@link ExpDataIterators.FileLinkDataIterator} for data iterator logical equivalent */ - TableInfo t = _sampleType.getTinfo(); - // Sample type uses FILE_LINK not FILE_ATTACHMENT, use convertTypes() to handle posted files - Path path = AssayFileWriter.getUploadDirectoryPath(c, SAMPLE_TYPE_FILE_DIRECTORY_NAME).toNioPathForWrite(); - convertTypes(user, c, validRowCopy, t, path); - if (t.getColumnNameSet().stream().anyMatch(validRowCopy::containsKey)) - { - keys = new Object[]{rowId}; - ret.putAll(Table.update(user, t, validRowCopy, t.getColumn(RowId.name()), keys, null, Level.DEBUG)); - } - - /** See {@link ExpDataIterators.SampleNameChangeDataIterator} for data iterator logical equivalent */ - ExpMaterialImpl sample = null; - if (hasNameChange) - { - sample = ExperimentServiceImpl.get().getExpMaterial(rowId); - if (sample != null) - ExperimentService.get().addObjectLegacyName(sample.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpMaterial.class), oldName, user); - } - - // update comment - /** See {@link ExpDataIterators.FlagDataIterator} for data iterator logical equivalent */ - if (row.containsKey(Flag.name()) || row.containsKey("comment")) - { - if (sample == null) - sample = ExperimentServiceImpl.get().getExpMaterial(rowId); - if (sample != null) - { - Object o = row.containsKey(Flag.name()) ? row.get(Flag.name()) : row.get("comment"); - String flag = Objects.toString(o, null); - sample.setComment(user, flag); - } - } - - // update aliases - /** See {@link ExpDataIterators.AliasDataIterator} for data iterator logical equivalent */ - if (row.containsKey(Alias.name())) - AliasInsertHelper.handleInsertUpdate(getContainer(), user, lsid, ExperimentService.get().getTinfoMaterialAliasMap(), row.get(Alias.name())); - - // search done in post-commit - - ret.put("lsid", lsid); - ret.put(AliquotedFromLSID.name(), oldRow.get(AliquotedFromLSID.name())); - ret.put(RowId.name(), rowId); // add RowId for SearchService - return ret; + throw new UnsupportedOperationException("_update() is no longer supported for samples"); } @Override @@ -1164,20 +911,6 @@ public List> deleteRows(User user, Container container, List return getMaterialStringValue(row, Name.name()); } - IntegerConverter _converter = new IntegerConverter(); - - private @Nullable Integer getMaterialIntegerValue(Map row, String columnName) - { - if (row != null) - { - Object o = row.get(columnName); - if (o != null) - return _converter.convert(Integer.class, o); - } - - return null; - } - private @Nullable Long getMaterialSourceId(Map row) { return MapUtils.getLong(row, MaterialSourceId.name()); From f678d15c758c137a069b4b09f520240ad2623d44 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Sun, 7 Dec 2025 16:17:15 -0800 Subject: [PATCH 40/62] New context --- .../experiment/api/SampleTypeUpdateServiceDI.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 40f2727d51b..65cca73e026 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -532,7 +532,6 @@ public List> updateRows( { results = scope.executeWithRetry(transaction -> { - var context = getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters); int index = 0; List> ret = new ArrayList<>(); @@ -548,14 +547,18 @@ public List> updateRows( var rowsToProcess = rows.subList(index, nextIndex); index = nextIndex; + var context = getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters); + var subRet = super._updateRowsUsingDIB(user, container, rowsToProcess, context, extraScriptContext); + + // we need to throw if we don't want executeWithRetry() attempt commit() + if (context.getErrors().hasErrors()) + throw new DbScope.RetryPassthroughException(context.getErrors()); + if (subRet != null) ret.addAll(subRet); } - // we need to throw if we don't want executeWithRetry() attempt commit() - if (context.getErrors().hasErrors()) - throw new DbScope.RetryPassthroughException(context.getErrors()); return ret; }); } From 8e14bb0bbec585de66169edca47c591ff267c8c4 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 8 Dec 2025 12:39:34 -0800 Subject: [PATCH 41/62] Clear ontology property cache with vocab changes --- .../labkey/experiment/ExpDataIterators.java | 54 ++++++++++++++++++- .../experiment/api/ExpSampleTypeTestCase.jsp | 1 - .../api/SampleTypeUpdateServiceDI.java | 13 +++-- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 64a3dc907ee..f2ee849f0aa 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -37,6 +37,7 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.CounterDefinition; +import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.ImportAliasable; @@ -64,6 +65,7 @@ import org.labkey.api.dataiterator.ValidatorIterator; import org.labkey.api.dataiterator.WrapperDataIterator; import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.PropertyType; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.api.ExpDataClass; @@ -82,6 +84,7 @@ import org.labkey.api.exp.api.NameExpressionOptionService; import org.labkey.api.exp.api.SampleTypeService; import org.labkey.api.exp.api.SimpleRunRecord; +import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.exp.property.PropertyService; import org.labkey.api.exp.query.AbstractExpSchema; import org.labkey.api.exp.query.DataClassUserSchema; @@ -2424,16 +2427,29 @@ public DataIterator getDataIterator(DataIteratorContext context) } } + Set vocabProps = PropertyService.get().findVocabularyProperties(_container, colNameMap.keySet()); + + // Ensure the property cache is cleared after vocabulary changes + if (isMergeOrUpdate && !vocabProps.isEmpty()) + { + var tx = _expTable.getSchema().getScope().getCurrentTransaction(); + if (tx != null) + tx.addCommitTask(OntologyManager::clearPropertyCache, DbScope.CommitTaskOption.POSTCOMMIT); + } + // Insert into exp.data then the provisioned table // Use embargo data iterator to ensure rows are committed before being sent along Issue 26082 (row at a time, reselect rowId) dib = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(dib, _expTable, _container) .setKeyColumns(keyColumns) .setDontUpdate(dontUpdate) - .setVocabularyProperties(PropertyService.get().findVocabularyProperties(_container, colNameMap.keySet())) + .setVocabularyProperties(vocabProps) .setAddlSkipColumns(_excludedColumns) .setCommitRowsBeforeContinuing(true) .setFailOnEmptyUpdate(false)); + if (isSample && isUpdateOnly) + dib = new EnsureLsidIteratorBuilder(dib, _expTable); + // pass in remap columns to help reconcile columns that may be aliased in the virtual table dib = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(dib, _propertiesTable, _container) .setKeyColumns(propertyKeyColumns) @@ -2472,6 +2488,42 @@ private DataIteratorBuilder getRootMaterialRowIdBuilder(DataIteratorBuilder dib) } } + /** + * This ensures the LSID column is included in the reselected row. + */ + private static class EnsureLsidIteratorBuilder implements DataIteratorBuilder + { + private final DataIteratorBuilder _in; + private final ColumnInfo _lsidCol; + + private EnsureLsidIteratorBuilder(@NotNull DataIteratorBuilder in, ExpTable expTable) + { + _in = in; + _lsidCol = expTable.getColumn(LSID.fieldKey()); + } + + @Override + public DataIterator getDataIterator(DataIteratorContext context) + { + DataIterator di = _in.getDataIterator(context); + if (!di.supportsGetExistingRecord()) + throw new IllegalArgumentException("DataIterator must support getExistingRecord()"); + + Map map = DataIteratorUtil.createColumnNameMap(di); + if (map.containsKey(LSID.name())) + return di; + + var ret = new SimpleTranslator(di, context); + ret.selectAll(); + ret.addColumn(_lsidCol, (Supplier) () -> { + Map old = ret.getExistingRecord(); + return old == null ? null : old.get(LSID.name()); + }); + + return ret; + } + } + private static class SampleUpdateOnlyValidatorsIteratorBuilder implements DataIteratorBuilder { private final Container _container; diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index d2f5fa15aac..7dff4129596 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -1088,7 +1088,6 @@ public void testSampleTypeWithVocabularyProperties() throws Exception oldKey.put("RowId", insertedSample.get(0).get("RowId")); oldKeys.add(oldKey); - // TODO: Figure out why LSID is no longer being pulled through on the row (even though this was previously invoking DIB pathway) var updatedSample = helper.updateRows(c, rowsToUpdate, oldKeys, sampleName, schema); assertEquals("Custom Property is not updated", updatedSampleType, OntologyManager.getPropertyObjects(c, updatedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 65cca73e026..2e8bf7756f4 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -524,32 +524,31 @@ public List> updateRows( return Collections.emptyList(); List> results; - DbScope scope = getSchema().getDbSchema().getScope(); Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; recordDataIteratorUsed(configParameters); try { - results = scope.executeWithRetry(transaction -> + results = getSchema().getDbSchema().getScope().executeWithRetry(tx -> { int index = 0; List> ret = new ArrayList<>(); while (index < rows.size()) { - var rowKeys = new CaseInsensitiveHashSet(rows.get(index).keySet()); + CaseInsensitiveHashSet rowKeys = new CaseInsensitiveHashSet(rows.get(index).keySet()); confirmAmountAndUnitsColumns(rowKeys); - var nextIndex = index + 1; + int nextIndex = index + 1; while (nextIndex < rows.size() && rowKeys.equals(new CaseInsensitiveHashSet(rows.get(nextIndex).keySet()))) nextIndex++; - var rowsToProcess = rows.subList(index, nextIndex); + List> rowsToProcess = rows.subList(index, nextIndex); index = nextIndex; - var context = getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters); + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters); - var subRet = super._updateRowsUsingDIB(user, container, rowsToProcess, context, extraScriptContext); + List> subRet = super._updateRowsUsingDIB(user, container, rowsToProcess, context, extraScriptContext); // we need to throw if we don't want executeWithRetry() attempt commit() if (context.getErrors().hasErrors()) From 59f9e27a4194fca37eb003c65c36c4e06d467ab7 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 8 Dec 2025 12:50:43 -0800 Subject: [PATCH 42/62] No longer include LSID in reselected rows by default --- .../labkey/experiment/ExpDataIterators.java | 40 ------------------- .../experiment/api/ExpSampleTypeTestCase.jsp | 27 +++++++------ 2 files changed, 15 insertions(+), 52 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index f2ee849f0aa..80b5df59128 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -37,7 +37,6 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.CounterDefinition; -import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.ImportAliasable; @@ -2447,9 +2446,6 @@ public DataIterator getDataIterator(DataIteratorContext context) .setCommitRowsBeforeContinuing(true) .setFailOnEmptyUpdate(false)); - if (isSample && isUpdateOnly) - dib = new EnsureLsidIteratorBuilder(dib, _expTable); - // pass in remap columns to help reconcile columns that may be aliased in the virtual table dib = LoggingDataIterator.wrap(new TableInsertDataIteratorBuilder(dib, _propertiesTable, _container) .setKeyColumns(propertyKeyColumns) @@ -2488,42 +2484,6 @@ private DataIteratorBuilder getRootMaterialRowIdBuilder(DataIteratorBuilder dib) } } - /** - * This ensures the LSID column is included in the reselected row. - */ - private static class EnsureLsidIteratorBuilder implements DataIteratorBuilder - { - private final DataIteratorBuilder _in; - private final ColumnInfo _lsidCol; - - private EnsureLsidIteratorBuilder(@NotNull DataIteratorBuilder in, ExpTable expTable) - { - _in = in; - _lsidCol = expTable.getColumn(LSID.fieldKey()); - } - - @Override - public DataIterator getDataIterator(DataIteratorContext context) - { - DataIterator di = _in.getDataIterator(context); - if (!di.supportsGetExistingRecord()) - throw new IllegalArgumentException("DataIterator must support getExistingRecord()"); - - Map map = DataIteratorUtil.createColumnNameMap(di); - if (map.containsKey(LSID.name())) - return di; - - var ret = new SimpleTranslator(di, context); - ret.selectAll(); - ret.addColumn(_lsidCol, (Supplier) () -> { - Map old = ret.getExistingRecord(); - return old == null ? null : old.get(LSID.name()); - }); - - return ret; - } - } - private static class SampleUpdateOnlyValidatorsIteratorBuilder implements DataIteratorBuilder { private final Container _container; diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index 7dff4129596..6abc053647c 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -1044,7 +1044,7 @@ public void testSampleTypeWithVocabularyProperties() throws Exception { User user = TestContext.get().getUser(); - String sampleName = "SamplesWithVocabularyProperties"; + String sampleTypeName = "SamplesWithVocabularyProperties"; String sampleType = "TypeA"; String updatedSampleType = "TypeB"; String sampleColor = "Blue"; @@ -1054,29 +1054,32 @@ public void testSampleTypeWithVocabularyProperties() throws Exception Map vocabularyPropertyURIs = helper.getVocabularyPropertyURIS(mockDomain); // create a sample type - createSampleType(sampleName, List.of(new GWTPropertyDescriptor("name", "string")), null); + createSampleType(sampleTypeName, List.of(new GWTPropertyDescriptor("name", "string")), null); // insert a sample + var sampleName = "TestSample"; ArrayListMap row = new ArrayListMap<>(); - row.put("name", "TestSample"); + row.put("name", sampleName); row.put(vocabularyPropertyURIs.get(helper.typePropertyName), sampleType); row.put(vocabularyPropertyURIs.get(helper.colorPropertyName), sampleColor); row.put(vocabularyPropertyURIs.get(helper.agePropertyName), null); // inserting a property with null value List> rows = helper.buildRows(row); UserSchema schema = getSampleSchema(); - var insertedSample = helper.insertRows(c, rows, sampleName, schema); + var insertedSample = helper.insertRows(c, rows, sampleTypeName, schema).get(0); + var sampleLsid = insertedSample.get("LSID").toString(); + var sampleRowId = insertedSample.get("RowId"); assertEquals("Custom Property is not inserted", sampleType, - OntologyManager.getPropertyObjects(c, insertedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); + OntologyManager.getPropertyObjects(c, sampleLsid).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); // Verifying property with null value is not inserted assertEquals("Property with null value is present.", 0, OntologyManager.getPropertyObjects(c, vocabularyPropertyURIs.get(helper.agePropertyName)).size()); // update inserted sample ArrayListMap rowToUpdate = new ArrayListMap<>(); - rowToUpdate.put("name", "TestSample"); - rowToUpdate.put("RowId", insertedSample.get(0).get("RowId")); + rowToUpdate.put("name", sampleName); + rowToUpdate.put("RowId", sampleRowId); rowToUpdate.put(vocabularyPropertyURIs.get(helper.typePropertyName), updatedSampleType); rowToUpdate.put(vocabularyPropertyURIs.get(helper.colorPropertyName), null); // nulling out existing property rowToUpdate.put(vocabularyPropertyURIs.get(helper.agePropertyName), sampleAge); //inserting a new property in update rows @@ -1084,20 +1087,20 @@ public void testSampleTypeWithVocabularyProperties() throws Exception List> oldKeys = new ArrayList<>(); ArrayListMap oldKey = new ArrayListMap<>(); - oldKey.put("name", "TestSample"); - oldKey.put("RowId", insertedSample.get(0).get("RowId")); + oldKey.put("name", sampleName); + oldKey.put("RowId", sampleRowId); oldKeys.add(oldKey); - var updatedSample = helper.updateRows(c, rowsToUpdate, oldKeys, sampleName, schema); + helper.updateRows(c, rowsToUpdate, oldKeys, sampleTypeName, schema); assertEquals("Custom Property is not updated", updatedSampleType, - OntologyManager.getPropertyObjects(c, updatedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); + OntologyManager.getPropertyObjects(c, sampleLsid).get(vocabularyPropertyURIs.get(helper.typePropertyName)).getStringValue()); // Verify property updated to a null value gets deleted assertEquals("Property with null value is present.", 0, OntologyManager.getPropertyObjects(c, vocabularyPropertyURIs.get(helper.colorPropertyName)).size()); // Verify property inserted during update rows in inserted assertEquals("New Property is not inserted with update rows", sampleAge, - OntologyManager.getPropertyObjects(c, updatedSample.get(0).get("LSID").toString()).get(vocabularyPropertyURIs.get(helper.agePropertyName)).getFloatValue().intValue()); + OntologyManager.getPropertyObjects(c, sampleLsid).get(vocabularyPropertyURIs.get(helper.agePropertyName)).getFloatValue().intValue()); } @Test From ad0d07ab5a73f7c4546a4e3844b29c9931b524a1 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 8 Dec 2025 13:18:31 -0800 Subject: [PATCH 43/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index 6a3d06f77f2..a5ed3e1bb5e 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-bnoC9ERAEL3uSbJC5ZFf8BbCF4KvPolN9gQ2jgY9IP57P06K5gGwt+IlrZH3inswglMjbHBzZ4oN/dD9T/uLfg==", + "version": "7.1.2-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-GwlLgMY4Hf7TYgSJW7TCxjboRSI0qg2eQD0f3eoQ3LxzJf+kfNAaRTLv1Pxv8n/QxXAUMpdP4Q/ViwAdIImFUw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index b87cc74fb27..54a8a125cb4 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index 0aca74f38f6..f23b5ac0561 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.1-fb-remove-sample-lsid.0", + "@labkey/components": "7.1.2-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-bnoC9ERAEL3uSbJC5ZFf8BbCF4KvPolN9gQ2jgY9IP57P06K5gGwt+IlrZH3inswglMjbHBzZ4oN/dD9T/uLfg==", + "version": "7.1.2-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-GwlLgMY4Hf7TYgSJW7TCxjboRSI0qg2eQD0f3eoQ3LxzJf+kfNAaRTLv1Pxv8n/QxXAUMpdP4Q/ViwAdIImFUw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 69d789ebf5c..4922d3f5323 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.1.1-fb-remove-sample-lsid.0", + "@labkey/components": "7.1.2-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index f63b5fe10dc..216261bcf8b 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-bnoC9ERAEL3uSbJC5ZFf8BbCF4KvPolN9gQ2jgY9IP57P06K5gGwt+IlrZH3inswglMjbHBzZ4oN/dD9T/uLfg==", + "version": "7.1.2-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-GwlLgMY4Hf7TYgSJW7TCxjboRSI0qg2eQD0f3eoQ3LxzJf+kfNAaRTLv1Pxv8n/QxXAUMpdP4Q/ViwAdIImFUw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index b9403be6b9f..ac4ecb5d8aa 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index f9bc40e5bc0..01fe3104450 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-bnoC9ERAEL3uSbJC5ZFf8BbCF4KvPolN9gQ2jgY9IP57P06K5gGwt+IlrZH3inswglMjbHBzZ4oN/dD9T/uLfg==", + "version": "7.1.2-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-GwlLgMY4Hf7TYgSJW7TCxjboRSI0qg2eQD0f3eoQ3LxzJf+kfNAaRTLv1Pxv8n/QxXAUMpdP4Q/ViwAdIImFUw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index 39d26f204ad..957aabf7328 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "7.1.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", From 7b1d975b5db5622b3f15c8825828cc6f7cfe9c6e Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 8 Dec 2025 19:40:11 -0800 Subject: [PATCH 44/62] MissingRowIds --- .../api/SampleTypeUpdateServiceDI.java | 111 ++++++++++++------ 1 file changed, 77 insertions(+), 34 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 2e8bf7756f4..f6c4a566ae8 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -143,6 +143,7 @@ import static org.labkey.api.exp.api.SampleTypeService.UNPROVIDED_VALUE_ERROR_MESSAGE_PATTERN; import static org.labkey.api.exp.query.ExpMaterialTable.Column.*; import static org.labkey.api.util.IntegerUtils.asLong; +import static org.labkey.experiment.ExpDataIterators.MultiDataTypeCrossProjectDataIterator.getRowIdNotAcceptedMessage; import static org.labkey.experiment.ExpDataIterators.incrementCounts; import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.insert; import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.rollup; @@ -284,12 +285,13 @@ private Pair, Set> getSampleParentsForRecalc(List> rows, DataIteratorContext context) { DataIterator it = etl.getDataIterator(context); + if (it == null || context.getErrors().hasErrors()) + return 0; try { if (null != rows) { - MapDataIterator maps = DataIteratorUtil.wrapMap(it, false); Map columnMap = DataIteratorUtil.createColumnNameMap(it); Integer parenRowIdToRecomputeCol = columnMap.get(ROOT_RECOMPUTE_ROWID_COL); @@ -399,7 +401,6 @@ public DataIteratorBuilder createImportDIB(User user, Container container, DataI return new ExpDataIterators.MultiDataTypeCrossProjectDataIteratorBuilder(user, container, data, context.isCrossTypeImport(), context.isCrossFolderImport(), _sampleType, true); DataIteratorBuilder dib = new ExpDataIterators.ExpMaterialDataIteratorBuilder(getQueryTable(), data, container, user); - dib = ((UpdateableTableInfo) getQueryTable()).persistRows(dib, context); dib = AttachmentDataIterator.getAttachmentDataIteratorBuilder(getQueryTable(), dib, user, context.getInsertOption().batch ? getAttachmentDirectory() : null, container, getAttachmentParentFactory()); dib = DetailedAuditLogDataIterator.getDataIteratorBuilder(getQueryTable(), dib, context.getInsertOption(), user, container, this::extractProvidedAmountsAndUnits); @@ -1098,8 +1099,12 @@ public Map> getExistingRows( throw new QueryUpdateServiceException("Either RowId or Name is required to get Sample Type Material."); } - if (!rowIdRowNumMap.isEmpty()) + Set missingRowIds; + if (rowIdRowNumMap.isEmpty()) + missingRowIds = Collections.emptySet(); + else { + missingRowIds = new HashSet<>(rowIdRowNumMap.keySet()); SimpleFilter filter = new SimpleFilter(RowId.fieldKey(), rowIdRowNumMap.keySet(), CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), container); Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); @@ -1107,17 +1112,17 @@ public Map> getExistingRows( { Long rowId = asLong(row.get(RowId.name())); Integer rowNum = rowIdRowNumMap.get(rowId); + missingRowIds.remove(rowId); sampleRows.put(rowNum, row); } } - Set allKeys; - + Set missingNames; if (nameRowNumMap.isEmpty()) - allKeys = Collections.emptySet(); + missingNames = Collections.emptySet(); else { - allKeys = new HashSet<>(nameRowNumMap.keySet()); + missingNames = new HashSet<>(nameRowNumMap.keySet()); SimpleFilter filter = new SimpleFilter(MaterialSourceId.fieldKey(), sampleTypeId); filter.addCondition(Name.fieldKey(), nameRowNumMap.keySet(), CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), container); @@ -1127,11 +1132,11 @@ public Map> getExistingRows( String name = (String) row.get(Name.name()); Integer rowNum = nameRowNumMap.get(name); sampleRows.put(rowNum, row); - allKeys.remove(name); + missingNames.remove(name); } } - if (verifyNoCrossFolderData && !allKeys.isEmpty()) + if (verifyNoCrossFolderData && (!missingNames.isEmpty() || !missingRowIds.isEmpty())) { // Issue 52922: cross-folder merge without Product Folders enabled silently ignores the cross-folder // row update. Use a relaxed container filter to find existing data from cross-containers. @@ -1141,18 +1146,35 @@ public Map> getExistingRows( if (!containerIds.isEmpty()) { - SimpleFilter filter = new SimpleFilter(MaterialSourceId.fieldKey(), sampleTypeId); - filter.addCondition(FieldKey.fromParts("Container"), containerIds, CompareType.IN); - filter.addCondition(Name.fieldKey(), allKeys, CompareType.IN); + if (!missingRowIds.isEmpty()) + { + SimpleFilter filter = new SimpleFilter(RowId.fieldKey(), missingRowIds, CompareType.IN); + filter.addCondition(FieldKey.fromParts("Container"), containerIds, CompareType.IN); + var row = new TableSelector(ExperimentService.get().getTinfoMaterial(), Sets.newCaseInsensitiveHashSet(RowId.name(), Name.name()), filter, null).setMaxRows(1).getMap(); + if (row != null) + throw new InvalidKeyException("Sample does not belong to " + container.getName() + " container: " + row.get(Name.name()) + " (" + row.get(RowId.name()) + ")."); + } - var row = new TableSelector(ExperimentService.get().getTinfoMaterial(), Sets.newCaseInsensitiveHashSet(Name.name()), filter, null).setMaxRows(1).getMap(); - if (row != null) - throw new InvalidKeyException("Sample does not belong to " + container.getName() + " container: " + row.get("name") + "."); + if (!missingNames.isEmpty()) + { + SimpleFilter filter = new SimpleFilter(MaterialSourceId.fieldKey(), sampleTypeId); + filter.addCondition(FieldKey.fromParts("Container"), containerIds, CompareType.IN); + filter.addCondition(Name.fieldKey(), missingNames, CompareType.IN); + + var row = new TableSelector(ExperimentService.get().getTinfoMaterial(), Sets.newCaseInsensitiveHashSet(Name.name()), filter, null).setMaxRows(1).getMap(); + if (row != null) + throw new InvalidKeyException("Sample does not belong to " + container.getName() + " container: " + row.get(Name.name()) + "."); + } } } - if (verifyExisting && !allKeys.isEmpty()) - throw new InvalidKeyException("Sample does not exist: " + allKeys.iterator().next() + "."); + if (verifyExisting) + { + if (!missingRowIds.isEmpty()) + throw new InvalidKeyException("Sample does not exist: (RowId) " + missingRowIds.iterator().next() + "."); + if (!missingNames.isEmpty()) + throw new InvalidKeyException("Sample does not exist: " + missingNames.iterator().next() + "."); + } // if contains domain fields, check for aliquot specific fields if (!queryTableInfo.getName().equalsIgnoreCase("material")) @@ -1389,7 +1411,10 @@ public PreTriggerDataIteratorBuilder(@NotNull ExpSampleTypeImpl sampleType, ExpM @Override public DataIterator getDataIterator(DataIteratorContext context) { - DataIterator source = LoggingDataIterator.wrap(builder.getDataIterator(context)); + DataIterator di = builder.getDataIterator(context); + if (di == null) + return null; // can happen if context has errors + boolean isMerge = context.getInsertOption() == InsertOption.MERGE; boolean isUpdate = context.getInsertOption() == InsertOption.UPDATE; @@ -1397,9 +1422,10 @@ public DataIterator getDataIterator(DataIteratorContext context) ColumnInfo containerColumn = materialTable.getColumn(materialTable.getContainerFieldKey()); String containerFieldLabel = containerColumn.getLabel(); var drop = new CaseInsensitiveHashSet(); - for (int i = 1; i <= source.getColumnCount(); i++) + var keysCheck = new CaseInsensitiveHashSet(); + for (int i = 1; i <= di.getColumnCount(); i++) { - String name = source.getColumnInfo(i).getName(); + String name = di.getColumnInfo(i).getName(); boolean isContainerField = name.equalsIgnoreCase(containerFieldLabel); if (!isContainerField) isContainerField = name.equalsIgnoreCase("Container") || name.equalsIgnoreCase("Folder"); @@ -1410,7 +1436,10 @@ public DataIterator getDataIterator(DataIteratorContext context) if (isCommentHeader(name)) continue; if (isNameHeader(name)) + { + keysCheck.add(Name.name()); continue; + } if (isDescriptionHeader(name)) continue; if (ExperimentService.isInputOutputColumn(name)) @@ -1429,24 +1458,37 @@ public DataIterator getDataIterator(DataIteratorContext context) continue; if (isExpMaterialColumn(RowId, name)) { + keysCheck.add(RowId.name()); if (isUpdate) continue; if (isMerge) - throw new IllegalArgumentException("RowId is not accepted when merging samples. Specify only the sample name instead."); + { + context.getErrors().addRowError(new ValidationException(getRowIdNotAcceptedMessage("merging"), RowId.name())); + return null; + } } + if (isExpMaterialColumn(LSID, name)) + keysCheck.add(LSID.name()); drop.add(name); } } + if ((isMerge || isUpdate) && keysCheck.size() == 1 && keysCheck.contains(LSID.name())) + { + String message = String.format("LSID is no longer accepted as a key for sample %s. Specify a RowId or Name instead.", isMerge ? "merge" : "update"); + context.getErrors().addRowError(new ValidationException(message, LSID.name())); + return null; + } + if (context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate)) drop.remove(LSID.name()); if (!drop.isEmpty()) - source = new DropColumnsDataIterator(source, drop); + di = new DropColumnsDataIterator(di, drop); - Map columnNameMap = DataIteratorUtil.createColumnNameMap(source); + Map columnNameMap = DataIteratorUtil.createColumnNameMap(di); if (isUpdate) { - SimpleTranslator addAliquotedFrom = new SimpleTranslator(source, context); + SimpleTranslator addAliquotedFrom = new SimpleTranslator(di, context); if (!columnNameMap.containsKey(AliquotedFromLSID.name())) addAliquotedFrom.addNullColumn(AliquotedFromLSID.name(), JdbcType.VARCHAR); @@ -1459,12 +1501,13 @@ public DataIterator getDataIterator(DataIteratorContext context) addAliquotedFrom.addNullColumn(PARENT_RECOMPUTE_NAME_COL, JdbcType.VARCHAR); addAliquotedFrom.selectAll(); - DataIterator di = new SampleUpdateAddColumnsDataIterator( - new CachingDataIterator(addAliquotedFrom), - materialTable, - sampleType.getRowId(), - getKeyColumnAlias(materialTable, columnNameMap) - ); + String keyColumnAlias = getKeyColumnAlias(materialTable, columnNameMap); + if (keyColumnAlias == null) + { + context.getErrors().addRowError(new ValidationException(String.format(DUPLICATE_COLUMN_IN_DATA_ERROR, RowId.name()))); + return null; + } + di = new SampleUpdateAddColumnsDataIterator(new CachingDataIterator(addAliquotedFrom), materialTable, sampleType.getRowId(), keyColumnAlias); di = new _SamplesCoerceDataIterator(di, context, sampleType, materialTable); context.setWithLookupRemapping(false); @@ -1474,7 +1517,7 @@ public DataIterator getDataIterator(DataIteratorContext context) // CoerceDataIterator to handle the lookup/alternatekeys functionality of loadRows(), // TODO: check if this covers all the functionality, in particular how is alternateKeyCandidates used? - DataIterator c = LoggingDataIterator.wrap(new _SamplesCoerceDataIterator(source, context, sampleType, materialTable)); + DataIterator c = LoggingDataIterator.wrap(new _SamplesCoerceDataIterator(di, context, sampleType, materialTable)); context.setWithLookupRemapping(false); SimpleTranslator addColumns = new SimpleTranslator(c, context); addColumns.setDebugName("add genId and other required columns"); @@ -1495,7 +1538,7 @@ public DataIterator getDataIterator(DataIteratorContext context) addColumns.addNullColumn(PARENT_RECOMPUTE_NAME_COL, JdbcType.VARCHAR); } - DataIterator di = LoggingDataIterator.wrap(addColumns); + di = LoggingDataIterator.wrap(addColumns); // Table Counters di = ExpDataIterators.CounterDataIteratorBuilder @@ -1508,7 +1551,7 @@ public DataIterator getDataIterator(DataIteratorContext context) return LoggingDataIterator.wrap(names); } - private static @NotNull String getKeyColumnAlias(TableInfo materialTable, @NotNull Map columnNameMap) + private static @Nullable String getKeyColumnAlias(TableInfo materialTable, @NotNull Map columnNameMap) { // Currently, SampleUpdateAddColumnsDataIterator is being called before a translator is invoked to // remap column labels to columns (e.g., "Row Id" -> "RowId"). Due to this, we need to search the @@ -1521,7 +1564,7 @@ public DataIterator getDataIterator(DataIteratorContext context) if (rowIdAliases.isEmpty()) return Name.name(); - throw new IllegalArgumentException(String.format(DUPLICATE_COLUMN_IN_DATA_ERROR, RowId.name())); + return null; } private static boolean isReservedHeader(String name) From 23eec7c1aba237d3227f1b897c1307f443214f08 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 8 Dec 2025 19:40:34 -0800 Subject: [PATCH 45/62] Handle context errors --- .../dataiterator/AttachmentDataIterator.java | 5 +- .../DetailedAuditLogDataIterator.java | 9 ++-- .../labkey/experiment/ExpDataIterators.java | 54 +++++++++++++++++-- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java index 4dc5b12e776..c8065c52077 100644 --- a/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/AttachmentDataIterator.java @@ -202,8 +202,11 @@ public static DataIteratorBuilder getAttachmentDataIteratorBuilder(TableInfo ti, throw new IllegalStateException("Originating data iterator is null"); DataIterator it = builder.getDataIterator(context); + if (it == null) + return null; // can happen if context has errors + Domain domain = ti.getDomain(); - if(domain == null) + if (domain == null) return it; // find attachment columns diff --git a/api/src/org/labkey/api/dataiterator/DetailedAuditLogDataIterator.java b/api/src/org/labkey/api/dataiterator/DetailedAuditLogDataIterator.java index 3f102575ab5..ec3696f2d8b 100644 --- a/api/src/org/labkey/api/dataiterator/DetailedAuditLogDataIterator.java +++ b/api/src/org/labkey/api/dataiterator/DetailedAuditLogDataIterator.java @@ -139,6 +139,10 @@ public static DataIteratorBuilder getDataIteratorBuilder(TableInfo queryTable, @ { return context -> { + DataIterator it = builder.getDataIterator(context); + if (it == null) + return null; // can happen if context has errors + AuditBehaviorType auditType = AuditBehaviorType.NONE; if (queryTable.supportsAuditTracking()) auditType = queryTable.getEffectiveAuditBehavior((AuditBehaviorType) context.getConfigParameter(AuditConfigs.AuditBehavior)); @@ -146,12 +150,12 @@ public static DataIteratorBuilder getDataIteratorBuilder(TableInfo queryTable, @ // Detailed auditing and not set to bulk load in ETL if (auditType == DETAILED && !context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad) && !context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.ByPassAudit)) { - DataIterator it = builder.getDataIterator(context); DataIterator in = DataIteratorUtil.wrapMap(it, true); return new DetailedAuditLogDataIterator(in, context, queryTable, insertOption.auditAction, user, container, extractProvidedValues); } + // Nothing to do, so just return input DataIterator - return builder.getDataIterator(context); + return it; }; } @@ -168,5 +172,4 @@ public boolean supportsGetExistingRecord() { return _data.supportsGetExistingRecord(); } - } diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 80b5df59128..771f7cf9944 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -216,6 +216,8 @@ public CounterDataIteratorBuilder(@NotNull DataIteratorBuilder in, Container con public DataIterator getDataIterator(DataIteratorContext context) { DataIterator pre = _in.getDataIterator(context); + if (pre == null) + return null; // can happen if context has errors SimpleTranslator counterTranslator = new SimpleTranslator(pre, context); counterTranslator.setDebugName("Counter Def"); @@ -349,6 +351,9 @@ public AliquotRollupDataIteratorBuilder(@NotNull DataIteratorBuilder in, Contain public DataIterator getDataIterator(DataIteratorContext context) { DataIterator pre = _in.getDataIterator(context); + if (pre == null) + return null; // can happen if context has errors + return LoggingDataIterator.wrap(new AliquotRollupDataIterator(pre, context, _container)); } } @@ -508,8 +513,11 @@ public AliasDataIteratorBuilder(@NotNull DataIteratorBuilder in, Container conta @Override public DataIterator getDataIterator(DataIteratorContext context) { - DataIterator pre = _in.getDataIterator(context); - return LoggingDataIterator.wrap(new AliasDataIterator(pre, context, _container, _user, _expAliasTable, _dataType, _isSample)); + DataIterator di = _in.getDataIterator(context); + if (di == null) + return null; // can happen if context has errors + + return LoggingDataIterator.wrap(new AliasDataIterator(di, context, _container, _user, _expAliasTable, _dataType, _isSample)); } } @@ -627,6 +635,9 @@ public AutoLinkToStudyDataIteratorBuilder(@NotNull DataIteratorBuilder in, UserS public DataIterator getDataIterator(DataIteratorContext context) { DataIterator pre = _in.getDataIterator(context); + if (pre == null) + return null; // can happen if context has errors + return LoggingDataIterator.wrap(new AutoLinkToStudyDataIterator(DataIteratorUtil.wrapMap(pre, false), _schema, _container, _user, _sampleType)); } } @@ -747,6 +758,9 @@ public FlagDataIteratorBuilder(@NotNull DataIteratorBuilder in, User user, boole public DataIterator getDataIterator(DataIteratorContext context) { DataIterator pre = _in.getDataIterator(context); + if (pre == null) + return null; // can happen if context has errors + return LoggingDataIterator.wrap(new FlagDataIterator(pre, context, _user, _isSample, _expObject, _container)); } } @@ -888,6 +902,9 @@ public DerivationDataIteratorBuilder(DataIteratorBuilder pre, Container containe public DataIterator getDataIterator(DataIteratorContext context) { DataIterator di = _pre.getDataIterator(context); + if (di == null) + return null; // can happen if context has errors + if (context.getConfigParameters().containsKey(SampleTypeUpdateServiceDI.Options.SkipDerivation)) return di; @@ -2104,6 +2121,9 @@ public SearchIndexIteratorBuilder(DataIteratorBuilder pre, Function map = DataIteratorUtil.createColumnNameMap(validate); @@ -2537,6 +2560,9 @@ public SampleNameChangeDataIteratorBuilder(@NotNull DataIteratorBuilder in, User public DataIterator getDataIterator(DataIteratorContext context) { DataIterator di = _in.getDataIterator(context); + if (di == null) + return null; // can happen if context has errors + return LoggingDataIterator.wrap(new SampleNameChangeDataIterator(di, context, _user, _canUpdateNames)); } } @@ -2640,7 +2666,7 @@ record TypeData( private final boolean _isCrossFolderUpdate; private final TSVWriter _tsvWriter; - public MultiDataTypeCrossProjectDataIterator(DataIterator di, DataIteratorContext context, Container container, User user, boolean isCrossType, boolean isCrossFolder, ExpObject dataType, boolean isSamples) + private MultiDataTypeCrossProjectDataIterator(DataIterator di, DataIteratorContext context, Container container, User user, boolean isCrossType, boolean isCrossFolder, ExpObject dataType, boolean isSamples) { super(di); _context = context; @@ -3300,6 +3326,11 @@ private void writeRowsToFile(TypeData typeData) _context.getErrors().addRowError(new ValidationException("Unable to write data for '" + typeData.dataType.getName() + "'.")); } } + + public static String getRowIdNotAcceptedMessage(String action) + { + return String.format("RowId is not accepted when %s samples. Specify only the sample name instead.", action); + } } public static class MultiDataTypeCrossProjectDataIteratorBuilder implements DataIteratorBuilder @@ -3326,8 +3357,18 @@ public MultiDataTypeCrossProjectDataIteratorBuilder(@NotNull User user, @NotNull @Override public DataIterator getDataIterator(DataIteratorContext context) { - DataIterator pre = _in.getDataIterator(context); - return LoggingDataIterator.wrap(new MultiDataTypeCrossProjectDataIterator(pre, context, _container, _user, _isCrossType, _isCrossFolder, _dataType, _isSamples)); + DataIterator di = _in.getDataIterator(context); + if (di == null) + return null; // can happen if context has errors + + Map map = DataIteratorUtil.createColumnNameMap(di); + if (_isSamples && map.get(RowId.name()) != null) + { + context.getErrors().addRowError(new ValidationException(MultiDataTypeCrossProjectDataIterator.getRowIdNotAcceptedMessage("importing"), RowId.name())); + return null; + } + + return LoggingDataIterator.wrap(new MultiDataTypeCrossProjectDataIterator(di, context, _container, _user, _isCrossType, _isCrossFolder, _dataType, _isSamples)); } } @@ -3354,6 +3395,9 @@ public SampleStatusCheckIteratorBuilder(@NotNull DataIteratorBuilder in, Contain public DataIterator getDataIterator(DataIteratorContext context) { DataIterator pre = _in.getDataIterator(context); + if (pre == null) + return null; // can happen if context has errors + return LoggingDataIterator.wrap(new SampleStatusCheckDataIterator(pre, context, _container)); } } From 7f89b63dbd5452a3d116944f09f03347e9555f5f Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 8 Dec 2025 19:40:38 -0800 Subject: [PATCH 46/62] Test updates --- .../experiment/api/ExpSampleTypeTestCase.jsp | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index 6abc053647c..c9c613b8424 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -789,15 +789,9 @@ public void testUpdateSomeParents() throws Exception rows.add(CaseInsensitiveHashMap.of("rowId", C1.getRowId(), "name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); rows.add(CaseInsensitiveHashMap.of("rowId", C4.getRowId(), "name", "C5", "MaterialInputs/Parent1Samples", null)); // intentionally mix up name - try - { - updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); - fail("Expected to throw exception"); - } - catch (Exception e) - { - assertThat(e.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); - } + updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); + assertThat(errors.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); + errors = new BatchValidationException(); } // Attempt to merge using "Row Id" label @@ -806,15 +800,37 @@ public void testUpdateSomeParents() throws Exception rows.add(CaseInsensitiveHashMap.of("Row Id", C1.getRowId(), "name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); rows.add(CaseInsensitiveHashMap.of("Row Id", C4.getRowId(), "name", "C5", "MaterialInputs/Parent1Samples", null)); // intentionally mix up name - try - { - updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); - fail("Expected to throw exception"); - } - catch (Exception e) - { - assertThat(e.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); - } + updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); + assertThat(errors.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); + errors = new BatchValidationException(); + } + + // Attempt to update using outdated "LSID" and do not specify any other keys + // Note: using try/catch here as updateRows() executes with retry which throws + // if validation exceptions are encountered. + try + { + rows.clear(); + rows.add(CaseInsensitiveHashMap.of("LSID", C1.getLSID(), "MaterialInputs/Parent1Samples", "P1-1")); + rows.add(CaseInsensitiveHashMap.of("LSID", C4.getLSID(), "MaterialInputs/Parent1Samples", null)); + + updateService.updateRows(user, c, rows, null, errors, null, null); + fail("Expected to throw exception"); + } + catch (Exception e) + { + assertThat(e.getMessage(), containsString("LSID is no longer accepted as a key for sample update")); + } + + // Attempt to merge using outdated "LSID" and do not specify any other keys + { + rows.clear(); + rows.add(CaseInsensitiveHashMap.of("LSID", C1.getLSID(), "MaterialInputs/Parent1Samples", "P1-1")); + rows.add(CaseInsensitiveHashMap.of("LSID", C4.getLSID(), "MaterialInputs/Parent1Samples", null)); + + updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); + assertThat(errors.getMessage(), containsString("LSID is no longer accepted as a key for sample merge")); + errors = new BatchValidationException(); } // now update the children with various types of modifications to the parentage From a67b923b6b5f328d0abf3a0c6adc9a911297e0c8 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 9 Dec 2025 10:08:03 -0800 Subject: [PATCH 47/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index a5ed3e1bb5e..d83d5c69562 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-GwlLgMY4Hf7TYgSJW7TCxjboRSI0qg2eQD0f3eoQ3LxzJf+kfNAaRTLv1Pxv8n/QxXAUMpdP4Q/ViwAdIImFUw==", + "version": "7.1.2-fb-remove-sample-lsid.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.1.tgz", + "integrity": "sha512-ztUqODGw1GX1Efc0zMBBldi/B25pO/B+Oty/Z5Dtw+kl5aKkH3CsykyO4ajZV+ZlldD6H4K5atxjwEH1IBDTQw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index 54a8a125cb4..f838de5c02c 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index f23b5ac0561..edaa7dfa84f 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.0", + "@labkey/components": "7.1.2-fb-remove-sample-lsid.1", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-GwlLgMY4Hf7TYgSJW7TCxjboRSI0qg2eQD0f3eoQ3LxzJf+kfNAaRTLv1Pxv8n/QxXAUMpdP4Q/ViwAdIImFUw==", + "version": "7.1.2-fb-remove-sample-lsid.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.1.tgz", + "integrity": "sha512-ztUqODGw1GX1Efc0zMBBldi/B25pO/B+Oty/Z5Dtw+kl5aKkH3CsykyO4ajZV+ZlldD6H4K5atxjwEH1IBDTQw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 4922d3f5323..457fc5e2929 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.0", + "@labkey/components": "7.1.2-fb-remove-sample-lsid.1", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index 216261bcf8b..d050f3da136 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-GwlLgMY4Hf7TYgSJW7TCxjboRSI0qg2eQD0f3eoQ3LxzJf+kfNAaRTLv1Pxv8n/QxXAUMpdP4Q/ViwAdIImFUw==", + "version": "7.1.2-fb-remove-sample-lsid.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.1.tgz", + "integrity": "sha512-ztUqODGw1GX1Efc0zMBBldi/B25pO/B+Oty/Z5Dtw+kl5aKkH3CsykyO4ajZV+ZlldD6H4K5atxjwEH1IBDTQw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index ac4ecb5d8aa..3c3f3b77507 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index 01fe3104450..4c0d43bd9f7 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-GwlLgMY4Hf7TYgSJW7TCxjboRSI0qg2eQD0f3eoQ3LxzJf+kfNAaRTLv1Pxv8n/QxXAUMpdP4Q/ViwAdIImFUw==", + "version": "7.1.2-fb-remove-sample-lsid.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.1.tgz", + "integrity": "sha512-ztUqODGw1GX1Efc0zMBBldi/B25pO/B+Oty/Z5Dtw+kl5aKkH3CsykyO4ajZV+ZlldD6H4K5atxjwEH1IBDTQw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index 957aabf7328..978bb22825f 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.0" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", From 7efc113721bc768a92cc1592a2d14f7f52dcc2f0 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 9 Dec 2025 11:39:04 -0800 Subject: [PATCH 48/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index d83d5c69562..cda928adb28 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.1.tgz", - "integrity": "sha512-ztUqODGw1GX1Efc0zMBBldi/B25pO/B+Oty/Z5Dtw+kl5aKkH3CsykyO4ajZV+ZlldD6H4K5atxjwEH1IBDTQw==", + "version": "7.1.2-fb-remove-sample-lsid.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.2.tgz", + "integrity": "sha512-i20EtNEs86VOwlAHM8NE8ttm1W97tB39a7qfiz+BttL7CC9+8e2TAOIKza1LxUgVamHORRkM2zkOrX8tjGJnkQ==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index f838de5c02c..e056354ea85 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index edaa7dfa84f..d65e5c46202 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.1", + "@labkey/components": "7.1.2-fb-remove-sample-lsid.2", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.1.tgz", - "integrity": "sha512-ztUqODGw1GX1Efc0zMBBldi/B25pO/B+Oty/Z5Dtw+kl5aKkH3CsykyO4ajZV+ZlldD6H4K5atxjwEH1IBDTQw==", + "version": "7.1.2-fb-remove-sample-lsid.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.2.tgz", + "integrity": "sha512-i20EtNEs86VOwlAHM8NE8ttm1W97tB39a7qfiz+BttL7CC9+8e2TAOIKza1LxUgVamHORRkM2zkOrX8tjGJnkQ==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 457fc5e2929..bbf7a52d41f 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.1", + "@labkey/components": "7.1.2-fb-remove-sample-lsid.2", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index d050f3da136..34674066921 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.1.tgz", - "integrity": "sha512-ztUqODGw1GX1Efc0zMBBldi/B25pO/B+Oty/Z5Dtw+kl5aKkH3CsykyO4ajZV+ZlldD6H4K5atxjwEH1IBDTQw==", + "version": "7.1.2-fb-remove-sample-lsid.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.2.tgz", + "integrity": "sha512-i20EtNEs86VOwlAHM8NE8ttm1W97tB39a7qfiz+BttL7CC9+8e2TAOIKza1LxUgVamHORRkM2zkOrX8tjGJnkQ==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index 3c3f3b77507..a1228e87e4c 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index 4c0d43bd9f7..a73ce0c532d 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.1.tgz", - "integrity": "sha512-ztUqODGw1GX1Efc0zMBBldi/B25pO/B+Oty/Z5Dtw+kl5aKkH3CsykyO4ajZV+ZlldD6H4K5atxjwEH1IBDTQw==", + "version": "7.1.2-fb-remove-sample-lsid.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.2.tgz", + "integrity": "sha512-i20EtNEs86VOwlAHM8NE8ttm1W97tB39a7qfiz+BttL7CC9+8e2TAOIKza1LxUgVamHORRkM2zkOrX8tjGJnkQ==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index 978bb22825f..1cb71ecb2af 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.1" + "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" }, "devDependencies": { "@labkey/build": "8.7.0", From 89fc75604a240c9090424f1c22de0e49a9e31544 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 9 Dec 2025 14:49:47 -0800 Subject: [PATCH 49/62] nits --- api/src/org/labkey/api/data/DataColumn.java | 4 +- .../org/labkey/api/data/DisplayColumn.java | 27 +-- .../experiment/api/ExpMaterialTableImpl.java | 191 ++++-------------- 3 files changed, 47 insertions(+), 175 deletions(-) diff --git a/api/src/org/labkey/api/data/DataColumn.java b/api/src/org/labkey/api/data/DataColumn.java index d2c2f593322..ff2c8c20738 100644 --- a/api/src/org/labkey/api/data/DataColumn.java +++ b/api/src/org/labkey/api/data/DataColumn.java @@ -259,8 +259,8 @@ public void addQueryFieldKeys(Set keys) { keys.add(_boundColumn.getFieldKey()); StringExpression effectiveURL = _boundColumn.getEffectiveURL(); - if (effectiveURL instanceof DetailsURL) - keys.addAll(((DetailsURL) effectiveURL).getFieldKeys()); + if (effectiveURL instanceof DetailsURL url) + keys.addAll(url.getFieldKeys()); } if (_displayColumn != null) keys.add(_displayColumn.getFieldKey()); diff --git a/api/src/org/labkey/api/data/DisplayColumn.java b/api/src/org/labkey/api/data/DisplayColumn.java index a9918cf3cd5..83c18d15813 100644 --- a/api/src/org/labkey/api/data/DisplayColumn.java +++ b/api/src/org/labkey/api/data/DisplayColumn.java @@ -58,7 +58,6 @@ import java.text.DecimalFormatSymbols; import java.text.Format; import java.util.ArrayList; -import java.util.Collection; import java.util.Date; import java.util.HashSet; import java.util.LinkedHashSet; @@ -283,23 +282,14 @@ public void addQueryFieldKeys(Set keys) else if (null != _url) se = StringExpressionFactory.createURL(_url); - if (se instanceof StringExpressionFactory.FieldKeyStringExpression) - { - Set fields = ((StringExpressionFactory.FieldKeyStringExpression)se).getFieldKeys(); - keys.addAll(fields); - } + if (se instanceof StringExpressionFactory.FieldKeyStringExpression expression) + keys.addAll(expression.getFieldKeys()); - if (_urlTitle instanceof StringExpressionFactory.FieldKeyStringExpression) - { - Set fields = ((StringExpressionFactory.FieldKeyStringExpression) _urlTitle).getFieldKeys(); - keys.addAll(fields); - } + if (_urlTitle instanceof StringExpressionFactory.FieldKeyStringExpression expression) + keys.addAll(expression.getFieldKeys()); - if (_textExpression instanceof StringExpressionFactory.FieldKeyStringExpression) - { - Set fields = ((StringExpressionFactory.FieldKeyStringExpression) _textExpression).getFieldKeys(); - keys.addAll(fields); - } + if (_textExpression instanceof StringExpressionFactory.FieldKeyStringExpression expression) + keys.addAll(expression.getFieldKeys()); _rowSpanner.addQueryColumns(keys); } @@ -351,13 +341,11 @@ public String getWidth() return _width; } - public void setNoWrap(boolean nowrap) { _nowrap = nowrap; } - // Ideally, this would just set the string... and defer creation of the Format object until render time, when we would // have a Container and other context. That would avoid creating multiple Formats per DisplayColumn. @Override @@ -369,7 +357,6 @@ public void setFormatString(String formatString) _tsvFormat = createFormat(formatString, tsvFormatSymbols); } - // java 7 changed to using infinity symbols for formatting, which is challenging for tsv import/export // use old school "Infinity" for now static public DecimalFormatSymbols tsvFormatSymbols = new DecimalFormatSymbols(); @@ -811,7 +798,7 @@ public void renderGridHeaderCell(RenderContext ctx, HtmlWriter out, String heade if (style == null) style = ""; - // 34871: Support for column display width + // Issue 34871: Support for column display width if (!isBlank(getWidth())) style += "; width:" + getWidth() + "px;"; diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index db5d6750395..11bc9a14ee8 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -39,8 +39,6 @@ import org.labkey.api.data.DataRegion; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.DisplayColumnFactory; import org.labkey.api.data.ForeignKey; import org.labkey.api.data.ImportAliasable; import org.labkey.api.data.JdbcType; @@ -337,10 +335,14 @@ public StringExpression getURL(ColumnInfo parent) { var columnInfo = wrapColumn(alias, _rootTable.getColumn(StoredAmount.name())); columnInfo.setDisplayColumnFactory(colInfo -> new SampleTypeAmountPrecisionDisplayColumn(colInfo, null)); + columnInfo.setConceptURI(NON_NEGATIVE_NUMBER_CONCEPT_URI); columnInfo.setDescription("The amount of this sample, in the base unit for the sample type's display unit (if defined), currently on hand."); - columnInfo.setUserEditable(false); + columnInfo.setHidden(true); columnInfo.setReadOnly(true); - columnInfo.setConceptURI(NON_NEGATIVE_NUMBER_CONCEPT_URI); + columnInfo.setShownInDetailsView(false); + columnInfo.setShownInInsertView(false); + columnInfo.setShownInUpdateView(false); + columnInfo.setUserEditable(false); columnInfo.setValidators(AMOUNT_RANGE_VALIDATORS); return columnInfo; } @@ -378,8 +380,12 @@ public StringExpression getURL(ColumnInfo parent) { var columnInfo = wrapColumn(alias, _rootTable.getColumn(Units.name())); columnInfo.setDescription("The units associated with the Stored Amount for this sample."); - columnInfo.setUserEditable(false); + columnInfo.setHidden(true); columnInfo.setReadOnly(true); + columnInfo.setShownInDetailsView(false); + columnInfo.setShownInInsertView(false); + columnInfo.setShownInUpdateView(false); + columnInfo.setUserEditable(false); return columnInfo; } case Units -> @@ -592,7 +598,10 @@ protected ContainerFilter getLookupContainerFilter() { var ret = wrapColumn(alias, _rootTable.getColumn(AliquotVolume.name())); ret.setLabel(RawAliquotVolume.label()); + ret.setHidden(true); ret.setShownInDetailsView(false); + ret.setShownInInsertView(false); + ret.setShownInUpdateView(false); return ret; } case AliquotVolume -> @@ -616,7 +625,10 @@ protected ContainerFilter getLookupContainerFilter() { var ret = wrapColumn(alias, _rootTable.getColumn(AvailableAliquotVolume.name())); ret.setLabel(RawAvailableAliquotVolume.label()); + ret.setHidden(true); ret.setShownInDetailsView(false); + ret.setShownInInsertView(false); + ret.setShownInUpdateView(false); return ret; } case AvailableAliquotVolume -> @@ -644,9 +656,12 @@ protected ContainerFilter getLookupContainerFilter() } case RawAliquotUnit -> { - var ret = wrapColumn(alias, _rootTable.getColumn("AliquotUnit")); - ret.setShownInDetailsView(false); + var ret = wrapColumn(alias, _rootTable.getColumn(AliquotUnit.name())); ret.setLabel(RawAliquotUnit.label()); + ret.setHidden(true); + ret.setShownInDetailsView(false); + ret.setShownInInsertView(false); + ret.setShownInUpdateView(false); return ret; } case AliquotUnit -> @@ -694,13 +709,7 @@ public MutableColumnInfo createPropertyColumn(String alias) var ret = wrapColumn(alias, _rootTable.getColumn(RowId.name())); if (_ss != null && _ss.getTinfo() != null) - { - ForeignKey fk = new QueryForeignKey.Builder(getUserSchema(), getLookupContainerFilter()) - .table(_ss.getTinfo()) - .key(RowId.name()) - .build(); - ret.setFk(fk); - } + ret.setFk(new QueryForeignKey.Builder(getUserSchema(), getLookupContainerFilter()).table(_ss.getTinfo()).key(RowId.name()).build()); ret.setIsUnselectable(true); ret.setDescription("A holder for any custom fields associated with this sample"); @@ -847,119 +856,11 @@ protected void populateColumns() addColumn(Units); defaultCols.add(Units.fieldKey()); - var rawAmountColumn = addColumn(RawAmount); - rawAmountColumn.setDisplayColumnFactory(new DisplayColumnFactory() - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new DataColumn(colInfo) - { - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - keys.add(StoredAmount.fieldKey()); - - } - }; - } - }); - rawAmountColumn.setHidden(true); - rawAmountColumn.setShownInDetailsView(false); - rawAmountColumn.setShownInInsertView(false); - rawAmountColumn.setShownInUpdateView(false); - - var rawUnitsColumn = addColumn(RawUnits); - rawUnitsColumn.setDisplayColumnFactory(new DisplayColumnFactory() - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new DataColumn(colInfo) - { - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - keys.add(Units.fieldKey()); - } - }; - } - }); - rawUnitsColumn.setHidden(true); - rawUnitsColumn.setShownInDetailsView(false); - rawUnitsColumn.setShownInInsertView(false); - rawUnitsColumn.setShownInUpdateView(false); - - var rawAliquotVolumeColumn = addColumn(RawAliquotVolume); - rawAliquotVolumeColumn.setDisplayColumnFactory(new DisplayColumnFactory() - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new DataColumn(colInfo) - { - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - keys.add(AliquotVolume.fieldKey()); - - } - }; - } - }); - rawAliquotVolumeColumn.setHidden(true); - rawAliquotVolumeColumn.setShownInDetailsView(false); - rawAliquotVolumeColumn.setShownInInsertView(false); - rawAliquotVolumeColumn.setShownInUpdateView(false); - - var rawAvailableAliquotVolumeColumn = addColumn(RawAvailableAliquotVolume); - rawAvailableAliquotVolumeColumn.setDisplayColumnFactory(new DisplayColumnFactory() - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new DataColumn(colInfo) - { - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - keys.add(AvailableAliquotVolume.fieldKey()); - - } - }; - } - }); - rawAvailableAliquotVolumeColumn.setHidden(true); - rawAvailableAliquotVolumeColumn.setShownInDetailsView(false); - rawAvailableAliquotVolumeColumn.setShownInInsertView(false); - rawAvailableAliquotVolumeColumn.setShownInUpdateView(false); - - var rawAliquotUnitColumn = addColumn(RawAliquotUnit); - rawAliquotUnitColumn.setDisplayColumnFactory(new DisplayColumnFactory() - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new DataColumn(colInfo) - { - @Override - public void addQueryFieldKeys(Set keys) - { - super.addQueryFieldKeys(keys); - keys.add(AliquotUnit.fieldKey()); - - } - }; - } - }); - rawAliquotUnitColumn.setHidden(true); - rawAliquotUnitColumn.setShownInDetailsView(false); - rawAliquotUnitColumn.setShownInInsertView(false); - rawAliquotUnitColumn.setShownInUpdateView(false); + addColumn(RawAmount).getRenderer().addQueryFieldKeys(new HashSet<>(Set.of(StoredAmount.fieldKey()))); + addColumn(RawUnits).getRenderer().addQueryFieldKeys(new HashSet<>(Set.of(Units.fieldKey()))); + addColumn(RawAliquotVolume).getRenderer().addQueryFieldKeys(new HashSet<>(Set.of(AliquotVolume.fieldKey()))); + addColumn(RawAvailableAliquotVolume).getRenderer().addQueryFieldKeys(new HashSet<>(Set.of(AvailableAliquotVolume.fieldKey()))); + addColumn(RawAliquotUnit).getRenderer().addQueryFieldKeys(new HashSet<>(Set.of(AliquotUnit.fieldKey()))); if (InventoryService.get() != null && (st == null || !st.isMedia())) defaultCols.addAll(InventoryService.get().addInventoryStatusColumns(st == null ? null : st.getMetricUnit(), this, getContainer(), _userSchema.getUser())); @@ -1009,10 +910,10 @@ public void addQueryFieldKeys(Set keys) addColumn(Properties); var colInputs = addColumn(Inputs); - addMethod("Inputs", new LineageMethod(colInputs, true), Set.of(colInputs.getFieldKey())); + addMethod(Inputs.name(), new LineageMethod(colInputs, true), Set.of(colInputs.getFieldKey())); var colOutputs = addColumn(Outputs); - addMethod("Outputs", new LineageMethod(colOutputs, false), Set.of(colOutputs.getFieldKey())); + addMethod(Outputs.name(), new LineageMethod(colOutputs, false), Set.of(colOutputs.getFieldKey())); addExpObjectMethod(); @@ -1151,7 +1052,14 @@ private void addSampleTypeColumns(ExpSampleType st, List visibleColumn if (idCols.contains(dp)) { propColumn.setNullable(false); - propColumn.setDisplayColumnFactory(new IdColumnRendererFactory()); + propColumn.setDisplayColumnFactory(c -> new DataColumn(c) + { + @Override + protected boolean isDisabledInput(RenderContext ctx) + { + return !super.isDisabledInput() && ctx.getMode() != DataRegion.MODE_INSERT; + } + }); } // Issue 38341: domain designer advanced settings 'show in default view' setting is not respected @@ -1645,29 +1553,6 @@ else if (rootField) return sql; } - private class IdColumnRendererFactory implements DisplayColumnFactory - { - @Override - public DisplayColumn createRenderer(ColumnInfo colInfo) - { - return new IdColumnRenderer(colInfo); - } - } - - private static class IdColumnRenderer extends DataColumn - { - public IdColumnRenderer(ColumnInfo col) - { - super(col); - } - - @Override - protected boolean isDisabledInput(RenderContext ctx) - { - return !super.isDisabledInput() && ctx.getMode() != DataRegion.MODE_INSERT; - } - } - private static class SampleTypeAmountDisplayColumn extends ExprColumn { public SampleTypeAmountDisplayColumn(TableInfo parent, String amountFieldName, String unitFieldName, String label, Set importAliases, Unit typeUnit) From 4c2f4e09668fb188862f922de813a7bf8cd7ab24 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 9 Dec 2025 16:17:28 -0800 Subject: [PATCH 50/62] Support cross-folder import rowId --- .../labkey/api/exp/query/ExpDataTable.java | 8 +- .../test/integration/SampleTypeCrud.ispec.ts | 84 ++++++++++++++++++- .../labkey/experiment/ExpDataIterators.java | 80 +++++++++++------- .../api/SampleTypeUpdateServiceDI.java | 3 +- 4 files changed, 140 insertions(+), 35 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpDataTable.java b/api/src/org/labkey/api/exp/query/ExpDataTable.java index b60aa86b18c..427ad473f9a 100644 --- a/api/src/org/labkey/api/exp/query/ExpDataTable.java +++ b/api/src/org/labkey/api/exp/query/ExpDataTable.java @@ -21,6 +21,7 @@ import org.labkey.api.exp.api.ExpExperiment; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.query.FieldKey; public interface ExpDataTable extends ExpTable { @@ -60,7 +61,12 @@ enum Column LastIndexed, Inputs, Outputs, - Properties + Properties; + + public FieldKey fieldKey() + { + return FieldKey.fromParts(name()); + } } void setExperiment(ExpExperiment experiment); diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index 32b65e02220..0faea797fe1 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -10,6 +10,7 @@ import { verifyRequiredLineageInsertUpdate } from './utils'; import { caseInsensitive, SAMPLE_TYPE_DESIGNER_ROLE } from '@labkey/components'; +const { importSample, insertRows } = ExperimentCRUDUtils; // @ts-expect-error process is not available in a browser environment const server = hookServer(process.env); @@ -314,7 +315,6 @@ describe('Sample Type Designer', () => { }); - describe('Import with update / merge', () => { it ("Issue 52922: Blank sample id in the file are getting ignored in update from file", async () => { const BLANK_KEY_UPDATE_ERROR = 'Name value not provided'; @@ -387,9 +387,89 @@ describe('Import with update / merge', () => { expect(successResp.text.indexOf('"success" : true') > -1).toBeTruthy(); }); + it('Support RowId lookup and renaming', async () => { + const dataType = SAMPLE_ALIQUOT_IMPORT_NO_NAME_PATTERN_NAME; + const initialName = 'RowIdLookupTest'; + const newName = 'RenamedViaRowId'; -}); + const rows = await insertRows(server, [{ + name: initialName, + description: 'Original Description' + }], 'samples', dataType, topFolderOptions, editorUserOptions); + + // Capture the generated RowId from the insert response + // Note: Assuming standard response structure where 'rows' array contains the returned data + const rowId = caseInsensitive(rows[0], 'rowId'); + expect(rowId).toBeDefined(); + + // Test: Update Description using ONLY RowId (Name column omitted) + // This validates that the importer can lookup by RowId alone + let updateTsv = `RowId\tDescription\n${rowId}\tUpdated Description via RowId`; + let resp = await importSample(server, updateTsv, dataType, 'UPDATE', topFolderOptions, editorUserOptions); + expect(resp.text.indexOf('"success" : true') > -1).toBeTruthy(); + + // Test: Update Name using RowId + Name (Renaming) + // This validates that providing both keys looks up by RowId and updates the Name + updateTsv = `RowId\tName\n${rowId}\t${newName}`; + resp = await importSample(server, updateTsv, dataType, 'UPDATE', topFolderOptions, editorUserOptions); + expect(resp.text.indexOf('"success" : true') > -1).toBeTruthy(); + + // Verify Rename: Attempt to update using the NEW Name + // If the rename worked, looking up by the new Name should succeed + updateTsv = `Name\tDescription\n${newName}\tDescription after rename`; + resp = await importSample(server, updateTsv, dataType, 'UPDATE', topFolderOptions, editorUserOptions); + expect(resp.text.indexOf('"success" : true') > -1).toBeTruthy(); + + // Verify Rename: Attempt to update using the OLD Name + // This should now fail because the record was renamed, proving the old key is gone + updateTsv = `Name\tDescription\n${initialName}\tShould fail`; + resp = await importSample(server, updateTsv, dataType, 'UPDATE', topFolderOptions, editorUserOptions); + expect(resp.text.indexOf('Sample does not exist') > -1).toBeTruthy(); + }); + it('Error when supplying RowId during MERGE', async () => { + const dataType = SAMPLE_ALIQUOT_IMPORT_NO_NAME_PATTERN_NAME; + const sampleName = 'MergeRowIdErrorTest'; + + const rows = await insertRows(server, [{ + name: sampleName, + description: 'created' + }], 'samples', dataType, topFolderOptions, editorUserOptions); + + const rowId = caseInsensitive(rows[0], 'rowId'); + expect(rowId).toBeDefined(); + // MERGE with RowId should fail + // Even if the name matches and rowId is correct, the presence of the column should trigger the error + const mergeTsv = `RowId\tName\tDescription\n${rowId}\t${sampleName}\tShould fail`; + const resp = await importSample(server, mergeTsv, dataType, 'MERGE', topFolderOptions, editorUserOptions); + + // Check for the specific error message + expect(resp.text.indexOf('RowId is not accepted when merging samples. Specify only the sample name instead.') > -1).toBeTruthy(); + }); + it('Error when supplying LSID without RowId or Name', async () => { + const dataType = SAMPLE_ALIQUOT_IMPORT_NO_NAME_PATTERN_NAME; + const sampleName = 'LsidKeyErrorTest'; + const LSID_UPDATE_ERROR = "LSID is no longer accepted as a key for sample update. Specify a RowId or Name instead."; + const LSID_MERGE_ERROR = "LSID is no longer accepted as a key for sample merge. Specify a RowId or Name instead."; + + const rows = await insertRows(server, [{ + name: sampleName, + description: 'created' + }], 'samples', dataType, topFolderOptions, editorUserOptions); + + const lsid = caseInsensitive(rows[0], 'lsid'); + expect(lsid).toBeDefined(); + + // UPDATE: LSID provided as key (Name/RowId missing) + const tsv = `LSID\tDescription\n${lsid}\tShould fail`; + let resp = await importSample(server, tsv, dataType, 'UPDATE', topFolderOptions, editorUserOptions); + expect(resp.text.indexOf(LSID_UPDATE_ERROR) > -1).toBeTruthy(); + + // MERGE: LSID provided as key (Name/RowId missing) + resp = await importSample(server, tsv, dataType, 'MERGE', topFolderOptions, editorUserOptions); + expect(resp.text.indexOf(LSID_MERGE_ERROR) > -1).toBeTruthy(); + }); +}); describe('Aliquot crud', () => { describe("SMAliquotImportExportTest", () => { diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 771f7cf9944..7f0a2212bd7 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -2641,7 +2641,7 @@ record TypeData( List fieldIndexes, Map dependencyIndexes, List dataRows, - List dataIds, + List dataIds, String headerRow, Map folderFiles ) { } @@ -2660,6 +2660,7 @@ record TypeData( private final Map> _typeFolderDataMap = new TreeMap<>(); private final Map> _orderDependencies = new HashMap<>(); private final int _dataIdIndex; + private final FieldKey _dataKey; private final Map> _idsPerType = new HashMap<>(); private final Map> _parentIdsPerType = new HashMap<>(); private final Map _containerMap = new CaseInsensitiveHashMap<>(); @@ -2678,7 +2679,36 @@ private MultiDataTypeCrossProjectDataIterator(DataIterator di, DataIteratorConte _isCrossFolder = isCrossFolder; Map map = DataIteratorUtil.createColumnNameMap(di); - _dataIdIndex = map.getOrDefault("Name", -1); + // Determine the dataId column + if (_isSamples) + { + int dataIdIndex = -1; + FieldKey dataKey = null; + + for (String dataId : RowId.namesAndLabels()) + { + if (map.containsKey(dataId)) + { + dataIdIndex = map.get(dataId); + dataKey = RowId.fieldKey(); + break; + } + } + + if (dataKey == null) + { + dataIdIndex = map.getOrDefault(Name.name(), -1); + dataKey = Name.fieldKey(); + } + + _dataIdIndex = dataIdIndex; + _dataKey = dataKey; + } + else + { + _dataIdIndex = map.getOrDefault(ExpDataTable.Column.Name.name(), -1); + _dataKey = ExpDataTable.Column.Name.fieldKey(); + } _tsvWriter = new TSVWriter() // Used to quote values with newline/tabs/quotes { @@ -2720,7 +2750,7 @@ protected int write() if (_folderColIndex != null || _isCrossFolderUpdate) { - ContainerFilter cf = ContainerFilter.current(container, user); + ContainerFilter cf; if (container.isProductFoldersEnabled()) { // Note that this is slightly different from our treatment of lookups: @@ -2731,11 +2761,14 @@ protected int write() else cf = new ContainerFilter.CurrentPlusProjectAndShared(container, user); } - Collection validContainerIds = cf.getIds(); + else + cf = ContainerFilter.current(container, user); + + Collection validContainerIds; if (cf instanceof ContainerFilter.ContainerFilterWithPermission cfp) - { validContainerIds = cfp.generateIds(container, context.getInsertOption().allowUpdate ? UpdatePermission.class : InsertPermission.class, null); - } + else + validContainerIds = cf.getIds(); if (validContainerIds != null) { @@ -2836,7 +2869,7 @@ public boolean next() throws BatchValidationException boolean hasCrossFolderImport = false; - // process the individual files + // process the individual files for (String key : importOrderKeys) { Map typeFolderData = _typeFolderDataMap.get(key); @@ -3201,7 +3234,7 @@ private void addDataRow(TypeData typeData) { _idsPerType.computeIfAbsent(typeData.dataType.getName(), k -> new HashSet<>()).add(data.toString()); if (_isCrossFolderUpdate) - typeData.dataIds.add(data.toString()); + typeData.dataIds.add(data); } // if the data represents a derivation dependency between types, and we're creating ids within the file, @@ -3219,7 +3252,7 @@ private void addDataRow(TypeData typeData) else if (index == _dataIdIndex && _isCrossFolderUpdate) { // Issue 52922: Samples with blank sample id in the file are getting ignored - throw new IllegalArgumentException("Name value not provided on row " + get(0)); + throw new IllegalArgumentException(_dataKey.getName() + " value not provided on row " + get(0)); } }); typeData.dataRows.add(StringUtils.join(dataRow, "\t")); @@ -3230,11 +3263,10 @@ private void writeRowsToFile(TypeData typeData) if (typeData.dataRows.isEmpty()) return; - // for cross folder import, write to further partitions + // for cross-folder import, write to further partitions if (_isCrossFolderUpdate) { ExpObject dataType = typeData.dataType; - Map> containerRows = new HashMap<>(); TableInfo tableInfo; @@ -3243,28 +3275,28 @@ private void writeRowsToFile(TypeData typeData) if (_isSamples) { filter = new SimpleFilter(MaterialSourceId.fieldKey(), dataType.getRowId()); - filter.addCondition(Name.fieldKey(), typeData.dataIds, CompareType.IN); + filter.addCondition(_dataKey, typeData.dataIds, CompareType.IN); tableInfo = ExperimentService.get().getTinfoMaterial(); } else { filter = new SimpleFilter(FieldKey.fromParts("ClassId"), dataType.getRowId()); - filter.addCondition(FieldKey.fromParts("Name"), typeData.dataIds, CompareType.IN); + filter.addCondition(_dataKey, typeData.dataIds, CompareType.IN); tableInfo = ExperimentService.get().getTinfoData(); } - Map[] rows = new TableSelector(tableInfo, Set.of("name", "container"), filter, null).getMapArray(); + Map[] rows = new TableSelector(tableInfo, Set.of(_dataKey.getName(), "container"), filter, null).getMapArray(); - Set notFoundIds = new HashSet<>(typeData.dataIds); + Set notFoundIds = new HashSet<>(typeData.dataIds); for (Map row : rows) { - String name = (String) row.get("name"); - notFoundIds.remove(name); + Object identifier = row.get(_dataKey.getName()); + notFoundIds.remove(identifier); String dataContainer = (String) row.get("container"); // could be updating the same data multiple times in a single import, the import will later be rejected List dataRowIds = IntStream.range(0, typeData.dataIds.size()).boxed() - .filter(i -> typeData.dataIds.get(i).equals(name)) + .filter(i -> typeData.dataIds.get(i).equals(identifier)) .toList(); containerRows.computeIfAbsent(dataContainer, k -> new ArrayList<>()).addAll(dataRowIds); } @@ -3326,11 +3358,6 @@ private void writeRowsToFile(TypeData typeData) _context.getErrors().addRowError(new ValidationException("Unable to write data for '" + typeData.dataType.getName() + "'.")); } } - - public static String getRowIdNotAcceptedMessage(String action) - { - return String.format("RowId is not accepted when %s samples. Specify only the sample name instead.", action); - } } public static class MultiDataTypeCrossProjectDataIteratorBuilder implements DataIteratorBuilder @@ -3361,13 +3388,6 @@ public DataIterator getDataIterator(DataIteratorContext context) if (di == null) return null; // can happen if context has errors - Map map = DataIteratorUtil.createColumnNameMap(di); - if (_isSamples && map.get(RowId.name()) != null) - { - context.getErrors().addRowError(new ValidationException(MultiDataTypeCrossProjectDataIterator.getRowIdNotAcceptedMessage("importing"), RowId.name())); - return null; - } - return LoggingDataIterator.wrap(new MultiDataTypeCrossProjectDataIterator(di, context, _container, _user, _isCrossType, _isCrossFolder, _dataType, _isSamples)); } } diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index f6c4a566ae8..5650e26f871 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -143,7 +143,6 @@ import static org.labkey.api.exp.api.SampleTypeService.UNPROVIDED_VALUE_ERROR_MESSAGE_PATTERN; import static org.labkey.api.exp.query.ExpMaterialTable.Column.*; import static org.labkey.api.util.IntegerUtils.asLong; -import static org.labkey.experiment.ExpDataIterators.MultiDataTypeCrossProjectDataIterator.getRowIdNotAcceptedMessage; import static org.labkey.experiment.ExpDataIterators.incrementCounts; import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.insert; import static org.labkey.experiment.api.SampleTypeServiceImpl.SampleChangeType.rollup; @@ -1463,7 +1462,7 @@ public DataIterator getDataIterator(DataIteratorContext context) continue; if (isMerge) { - context.getErrors().addRowError(new ValidationException(getRowIdNotAcceptedMessage("merging"), RowId.name())); + context.getErrors().addRowError(new ValidationException("RowId is not accepted when merging samples. Specify only the sample name instead.", RowId.name())); return null; } } From b6043585ae3842d7f82e89e3fde7b0b34ae29e9a Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 9 Dec 2025 21:12:40 -0800 Subject: [PATCH 51/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index cda928adb28..f6dd2a1b668 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.2", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.2.tgz", - "integrity": "sha512-i20EtNEs86VOwlAHM8NE8ttm1W97tB39a7qfiz+BttL7CC9+8e2TAOIKza1LxUgVamHORRkM2zkOrX8tjGJnkQ==", + "version": "7.2.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-0sD41qrQFoJhRr8OBCvSlFRyG/+dgSGwy/t7CyuDV6B7aowsvqcXHgleQ8VpqNrSrxXOE7RgPjYROnzNpq7L2g==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index e056354ea85..0e6613c8454 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index d65e5c46202..b32cfea4c6b 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.2", + "@labkey/components": "7.2.1-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.2", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.2.tgz", - "integrity": "sha512-i20EtNEs86VOwlAHM8NE8ttm1W97tB39a7qfiz+BttL7CC9+8e2TAOIKza1LxUgVamHORRkM2zkOrX8tjGJnkQ==", + "version": "7.2.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-0sD41qrQFoJhRr8OBCvSlFRyG/+dgSGwy/t7CyuDV6B7aowsvqcXHgleQ8VpqNrSrxXOE7RgPjYROnzNpq7L2g==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index bbf7a52d41f..b32088654e8 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.2", + "@labkey/components": "7.2.1-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index 34674066921..a5346d1c930 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.2", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.2.tgz", - "integrity": "sha512-i20EtNEs86VOwlAHM8NE8ttm1W97tB39a7qfiz+BttL7CC9+8e2TAOIKza1LxUgVamHORRkM2zkOrX8tjGJnkQ==", + "version": "7.2.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-0sD41qrQFoJhRr8OBCvSlFRyG/+dgSGwy/t7CyuDV6B7aowsvqcXHgleQ8VpqNrSrxXOE7RgPjYROnzNpq7L2g==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index a1228e87e4c..9c3fd6af813 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index a73ce0c532d..92481f54028 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.1.2-fb-remove-sample-lsid.2", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.1.2-fb-remove-sample-lsid.2.tgz", - "integrity": "sha512-i20EtNEs86VOwlAHM8NE8ttm1W97tB39a7qfiz+BttL7CC9+8e2TAOIKza1LxUgVamHORRkM2zkOrX8tjGJnkQ==", + "version": "7.2.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-0sD41qrQFoJhRr8OBCvSlFRyG/+dgSGwy/t7CyuDV6B7aowsvqcXHgleQ8VpqNrSrxXOE7RgPjYROnzNpq7L2g==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index 1cb71ecb2af..baf7679dbfa 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "7.1.2-fb-remove-sample-lsid.2" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", From 0002384956486860f97d6994302c7259a3844799 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 9 Dec 2025 22:08:33 -0800 Subject: [PATCH 52/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index f6dd2a1b668..10924160f1d 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.2.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-0sD41qrQFoJhRr8OBCvSlFRyG/+dgSGwy/t7CyuDV6B7aowsvqcXHgleQ8VpqNrSrxXOE7RgPjYROnzNpq7L2g==", + "version": "7.2.1-fb-remove-sample-lsid.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.1.tgz", + "integrity": "sha512-nmJgZoIKypCKYRrsHqXJPD1krmjO4omqqr1i0ifV813FzfgcVIDH2ogZr7/Oesp6yVPVD195Qfx6fSM/IXcsiA==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index 0e6613c8454..c06b31a2837 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index b32cfea4c6b..83c8faf57f5 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.0", + "@labkey/components": "7.2.1-fb-remove-sample-lsid.1", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.2.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-0sD41qrQFoJhRr8OBCvSlFRyG/+dgSGwy/t7CyuDV6B7aowsvqcXHgleQ8VpqNrSrxXOE7RgPjYROnzNpq7L2g==", + "version": "7.2.1-fb-remove-sample-lsid.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.1.tgz", + "integrity": "sha512-nmJgZoIKypCKYRrsHqXJPD1krmjO4omqqr1i0ifV813FzfgcVIDH2ogZr7/Oesp6yVPVD195Qfx6fSM/IXcsiA==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index b32088654e8..2df3aa528fb 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.0", + "@labkey/components": "7.2.1-fb-remove-sample-lsid.1", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index a5346d1c930..9e0243f98a1 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.2.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-0sD41qrQFoJhRr8OBCvSlFRyG/+dgSGwy/t7CyuDV6B7aowsvqcXHgleQ8VpqNrSrxXOE7RgPjYROnzNpq7L2g==", + "version": "7.2.1-fb-remove-sample-lsid.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.1.tgz", + "integrity": "sha512-nmJgZoIKypCKYRrsHqXJPD1krmjO4omqqr1i0ifV813FzfgcVIDH2ogZr7/Oesp6yVPVD195Qfx6fSM/IXcsiA==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index 9c3fd6af813..f1cd7ae6614 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index 92481f54028..1ac71c8b765 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.2.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-0sD41qrQFoJhRr8OBCvSlFRyG/+dgSGwy/t7CyuDV6B7aowsvqcXHgleQ8VpqNrSrxXOE7RgPjYROnzNpq7L2g==", + "version": "7.2.1-fb-remove-sample-lsid.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.1.tgz", + "integrity": "sha512-nmJgZoIKypCKYRrsHqXJPD1krmjO4omqqr1i0ifV813FzfgcVIDH2ogZr7/Oesp6yVPVD195Qfx6fSM/IXcsiA==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index baf7679dbfa..61f17bfd282 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" }, "devDependencies": { "@labkey/build": "8.7.0", From 7c1354e37e3f908436c61e1b4441262bac332164 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Tue, 9 Dec 2025 23:36:22 -0800 Subject: [PATCH 53/62] Convert key type --- .../labkey/experiment/ExpDataIterators.java | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 7f0a2212bd7..59e9f88e984 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -40,6 +40,7 @@ import org.labkey.api.data.DbScope; import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.ImportAliasable; +import org.labkey.api.data.JdbcType; import org.labkey.api.data.NameGenerator; import org.labkey.api.data.RemapCache; import org.labkey.api.data.SimpleFilter; @@ -2661,6 +2662,7 @@ record TypeData( private final Map> _orderDependencies = new HashMap<>(); private final int _dataIdIndex; private final FieldKey _dataKey; + private final boolean _dataKeyIsNumeric; private final Map> _idsPerType = new HashMap<>(); private final Map> _parentIdsPerType = new HashMap<>(); private final Map _containerMap = new CaseInsensitiveHashMap<>(); @@ -2680,34 +2682,40 @@ private MultiDataTypeCrossProjectDataIterator(DataIterator di, DataIteratorConte Map map = DataIteratorUtil.createColumnNameMap(di); // Determine the dataId column - if (_isSamples) { - int dataIdIndex = -1; - FieldKey dataKey = null; + int index; + FieldKey dataKey; + boolean isNumeric; - for (String dataId : RowId.namesAndLabels()) + if (_isSamples) { - if (map.containsKey(dataId)) + var foundId = RowId.namesAndLabels().stream() + .filter(map::containsKey) + .findFirst(); + + if (foundId.isPresent()) { - dataIdIndex = map.get(dataId); + index = map.get(foundId.get()); dataKey = RowId.fieldKey(); - break; + isNumeric = true; + } + else + { + index = map.getOrDefault(Name.name(), -1); + dataKey = Name.fieldKey(); + isNumeric = false; } } - - if (dataKey == null) + else { - dataIdIndex = map.getOrDefault(Name.name(), -1); - dataKey = Name.fieldKey(); + index = map.getOrDefault(ExpDataTable.Column.Name.name(), -1); + dataKey = ExpDataTable.Column.Name.fieldKey(); + isNumeric = false; } - _dataIdIndex = dataIdIndex; + _dataIdIndex = index; _dataKey = dataKey; - } - else - { - _dataIdIndex = map.getOrDefault(ExpDataTable.Column.Name.name(), -1); - _dataKey = ExpDataTable.Column.Name.fieldKey(); + _dataKeyIsNumeric = isNumeric; } _tsvWriter = new TSVWriter() // Used to quote values with newline/tabs/quotes @@ -3232,9 +3240,10 @@ private void addDataRow(TypeData typeData) { if (index == _dataIdIndex) { - _idsPerType.computeIfAbsent(typeData.dataType.getName(), k -> new HashSet<>()).add(data.toString()); + String dataString = data.toString(); + _idsPerType.computeIfAbsent(typeData.dataType.getName(), k -> new HashSet<>()).add(dataString); if (_isCrossFolderUpdate) - typeData.dataIds.add(data); + typeData.dataIds.add(_dataKeyIsNumeric ? JdbcType.BIGINT.convert(data) : dataString); } // if the data represents a derivation dependency between types, and we're creating ids within the file, @@ -3290,7 +3299,8 @@ private void writeRowsToFile(TypeData typeData) Set notFoundIds = new HashSet<>(typeData.dataIds); for (Map row : rows) { - Object identifier = row.get(_dataKey.getName()); + Object raw = row.get(_dataKey.getName()); + Object identifier = _dataKeyIsNumeric ? asLong(raw) : raw; notFoundIds.remove(identifier); String dataContainer = (String) row.get("container"); // could be updating the same data multiple times in a single import, the import will later be rejected From db05b445ff81947db29256bb84ff150e5612bb33 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 10 Dec 2025 13:10:43 -0800 Subject: [PATCH 54/62] Do not update MaterialSourceId --- .../api/audit/SampleTimelineAuditEvent.java | 5 +- .../api/exp/query/ExpMaterialTable.java | 4 + .../test/integration/SampleTypeCrud.ispec.ts | 46 ++++++++++ .../labkey/experiment/ExpDataIterators.java | 86 ++++++++++++------- .../experiment/api/ExpMaterialTableImpl.java | 2 +- .../api/SampleTypeUpdateServiceDI.java | 14 +-- 6 files changed, 117 insertions(+), 40 deletions(-) diff --git a/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java b/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java index 3ed3637595f..9bc6b30f0e8 100644 --- a/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java +++ b/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java @@ -15,8 +15,7 @@ import static org.labkey.api.audit.AuditHandler.DELTA_PROVIDED_DATA_PREFIX; import static org.labkey.api.audit.AuditHandler.PROVIDED_DATA_PREFIX; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.*; public class SampleTimelineAuditEvent extends DetailedAuditTypeEvent { @@ -26,7 +25,7 @@ public class SampleTimelineAuditEvent extends DetailedAuditTypeEvent public static final String AMOUNT_AND_UNIT_UPGRADE_COMMENT = "Storage amount unit conversion to base unit during upgrade script."; public static final Set EXCLUDED_DETAIL_FIELDS = Set.of( - "AvailableAliquotVolume", "AvailableAliquotCount", "AliquotCount", "AliquotVolume", "AliquotUnit", + AvailableAliquotVolume.name(), AvailableAliquotCount.name(), AliquotCount.name(), AliquotVolume.name(), AliquotUnit.name(), PROVIDED_DATA_PREFIX + StoredAmount.name(), PROVIDED_DATA_PREFIX + Units.name(), DELTA_PROVIDED_DATA_PREFIX + StoredAmount.name() + DELTA_PROVIDED_DATA_PREFIX + Units.name()); diff --git a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java index d8f4c9af894..11ca4dd9e69 100644 --- a/api/src/org/labkey/api/exp/query/ExpMaterialTable.java +++ b/api/src/org/labkey/api/exp/query/ExpMaterialTable.java @@ -34,6 +34,7 @@ enum Column AliquotedFromLSID, AvailableAliquotCount, AvailableAliquotVolume(true), + CpasType, // database table only Created, CreatedBy, Description, @@ -48,6 +49,7 @@ enum Column Modified, ModifiedBy, Name, + ObjectId, // database table only Outputs, Properties, Property, @@ -60,10 +62,12 @@ enum Column RootMaterialRowId, RowId, Run, + RunId, // database table only RunApplication, RunApplicationOutput, SampleSet, SampleState, + SourceApplicationId, // database table only SourceApplicationInput, SourceProtocolApplication, SourceProtocolLSID, diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index 0faea797fe1..f1f2dec9b84 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -469,6 +469,52 @@ describe('Import with update / merge', () => { resp = await importSample(server, tsv, dataType, 'MERGE', topFolderOptions, editorUserOptions); expect(resp.text.indexOf(LSID_MERGE_ERROR) > -1).toBeTruthy(); }); + it('MaterialSourceId is immutable during update', async () => { + // Arrange + const firstSampleType = SAMPLE_ALIQUOT_IMPORT_TYPE_NAME; + const secondSampleType = SAMPLE_ALIQUOT_IMPORT_NO_NAME_PATTERN_NAME; + const [firstSample] = await insertRows(server, [{ name: 'FL-1', description: 'Yolo' }], 'samples', firstSampleType, topFolderOptions, editorUserOptions); + const [secondSample] = await insertRows(server, [{ name: 'SP-10', description: 'Hello' }], 'samples', secondSampleType, topFolderOptions, editorUserOptions); + const firstRowId = caseInsensitive(firstSample, 'rowId'); + const secondRowId = caseInsensitive(secondSample, 'rowId'); + + // Fetch sample type rowIds + let resp = await server.post('query', 'selectRows', { + schemaName: 'exp', + queryName: 'SampleSets', + 'query.columns': 'RowId, Name', + 'query.Name~in': [firstSampleType, secondSampleType].join(';'), + 'query.sort': 'Name', + }, { ...topFolderOptions, ...adminOptions }).expect(successfulResponse); + const firstSampleTypeRowId = caseInsensitive(resp.body.rows[0], 'RowId'); + const secondSampleTypeRowId = caseInsensitive(resp.body.rows[1], 'RowId'); + + let tsv = 'RowId\tSampleType\tDescription\n'; + tsv += `${firstRowId}\t${firstSampleType}\tShould be FL-1\n`; + tsv += `${secondRowId}\t${secondSampleType}\tShould be SP-10\n`; + + // Act + resp = await importSample(server, tsv, firstSampleType, 'UPDATE', topFolderOptions, editorUserOptions); + expect(resp.body.success).toEqual(true); + expect(resp.body.rowCount).toEqual(2); + + // Assert + // Verify that the MaterialSourceId is not altered for these rows + resp = await server.post('query', 'selectRows', { + schemaName: 'exp', + queryName: 'materials', + 'query.columns': 'RowId, Name, MaterialSourceId, Description', + 'query.sort': 'RowId', + }, { ...topFolderOptions, ...adminOptions }).expect(successfulResponse); + + expect(caseInsensitive(resp.body.rows[0], 'Name')).toEqual('FL-1'); + expect(caseInsensitive(resp.body.rows[0], 'MaterialSourceId')).toEqual(firstSampleTypeRowId); + expect(caseInsensitive(resp.body.rows[0], 'Description')).toEqual('Should be FL-1'); + + expect(caseInsensitive(resp.body.rows[1], 'Name')).toEqual('SP-10'); + expect(caseInsensitive(resp.body.rows[1], 'MaterialSourceId')).toEqual(secondSampleTypeRowId); + expect(caseInsensitive(resp.body.rows[1], 'Description')).toEqual('Should be SP-10'); + }) }); describe('Aliquot crud', () => { diff --git a/experiment/src/org/labkey/experiment/ExpDataIterators.java b/experiment/src/org/labkey/experiment/ExpDataIterators.java index 59e9f88e984..91329240e4d 100644 --- a/experiment/src/org/labkey/experiment/ExpDataIterators.java +++ b/experiment/src/org/labkey/experiment/ExpDataIterators.java @@ -15,6 +15,7 @@ */ package org.labkey.experiment; +import org.apache.commons.beanutils.ConversionException; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Strings; @@ -2294,14 +2295,32 @@ public boolean next() throws BatchValidationException } } - public static final Set NOT_FOR_UPDATE = Sets.newCaseInsensitiveHashSet( - ExpDataTable.Column.RowId.toString(), - ExpDataTable.Column.LSID.toString(), - ExpDataTable.Column.Created.toString(), - ExpDataTable.Column.CreatedBy.toString(), - AliquotedFromLSID.toString(), - RootMaterialRowId.toString(), - "genId"); + // Common fields in both exp.data and exp.material that cannot be updated + private static final Set COMMON_NOT_FOR_UPDATE = CaseInsensitiveHashSet.of( + Created.name(), + CreatedBy.name(), + LSID.name(), + RowId.name(), + "genId" + ); + + public static final Set DATA_NOT_FOR_UPDATE; + public static final Set MATERIAL_NOT_FOR_UPDATE; + + static { + DATA_NOT_FOR_UPDATE = COMMON_NOT_FOR_UPDATE; + + Set materialNotForUpdate = Sets.newCaseInsensitiveHashSet(COMMON_NOT_FOR_UPDATE); + materialNotForUpdate.addAll(CaseInsensitiveHashSet.of( + AliquotCount.name(), + AliquotedFromLSID.name(), + AliquotVolume.name(), + AvailableAliquotCount.name(), + AvailableAliquotVolume.name(), + RootMaterialRowId.name() + )); + MATERIAL_NOT_FOR_UPDATE = Collections.unmodifiableSet(materialNotForUpdate); + } public static class PersistDataIteratorBuilder implements DataIteratorBuilder { @@ -2311,7 +2330,7 @@ public static class PersistDataIteratorBuilder implements DataIteratorBuilder private final ExpObject _dataTypeObject; private final Container _container; private final User _user; - private final Set _excludedColumns = new HashSet<>(List.of("generated","runId","sourceapplicationid")); // generated has database DEFAULT 0 + private final Set _excludedColumns = CaseInsensitiveHashSet.of("generated", RunId.name(), SourceApplicationId.name()); // generated has database DEFAULT 0 private String _fileLinkDirectory = null; Function _indexFunction; @@ -2376,12 +2395,14 @@ public DataIterator getDataIterator(DataIteratorContext context) if (colNameMap.containsKey(Alias.name())) step1.addColumn(ExperimentService.ALIASCOLUMNALIAS, colNameMap.get(Alias.name())); // see AliasDataIteratorBuilder - CaseInsensitiveHashSet dontUpdate = new CaseInsensitiveHashSet(NOT_FOR_UPDATE); - if (isUpdateOnly) + CaseInsensitiveHashSet dontUpdate = new CaseInsensitiveHashSet(isSample ? MATERIAL_NOT_FOR_UPDATE : DATA_NOT_FOR_UPDATE); + if (isMergeOrUpdate) { - dontUpdate.add("objectid"); - dontUpdate.add("cpastype"); - dontUpdate.add("lastindexed"); + // Common fields in both exp.data and exp.material that cannot be updated + dontUpdate.addAll(CpasType.name(), ObjectId.name()); + + if (isSample) + dontUpdate.add(MaterialSourceId.name()); } CaseInsensitiveHashSet keyColumns = new CaseInsensitiveHashSet(); @@ -2404,25 +2425,17 @@ public DataIterator getDataIterator(DataIteratorContext context) dontUpdate.add(Name.name()); dontUpdate.addAll(((ExpMaterialTableImpl) _expTable).getUniqueIdFields()); - dontUpdate.addAll( - RootMaterialRowId.name(), - AliquotedFromLSID.name(), - AliquotCount.name(), - AliquotVolume.name(), - AvailableAliquotCount.name(), - AvailableAliquotVolume.name() - ); } else { if (isMergeOrUpdate) { - boolean isUpdateUsingLsid = isUpdateOnly && colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); - if (isUpdateUsingLsid) - { - if (!canUpdateNames) - dontUpdate.add(ExpDataTable.Column.Name.name()); - } + boolean isUpdateUsingLsid = isUpdateOnly && + colNameMap.containsKey(ExpDataTable.Column.LSID.name()) && + context.getConfigParameterBoolean(ExperimentService.QueryOptions.UseLsidForUpdate); + + if (isUpdateUsingLsid && !canUpdateNames) + dontUpdate.add(ExpDataTable.Column.Name.name()); } } @@ -3243,7 +3256,22 @@ private void addDataRow(TypeData typeData) String dataString = data.toString(); _idsPerType.computeIfAbsent(typeData.dataType.getName(), k -> new HashSet<>()).add(dataString); if (_isCrossFolderUpdate) - typeData.dataIds.add(_dataKeyIsNumeric ? JdbcType.BIGINT.convert(data) : dataString); + { + if (_dataKeyIsNumeric) + { + try + { + typeData.dataIds.add(JdbcType.BIGINT.convert(data)); + } + catch (ConversionException e) + { + _context.getErrors().addRowError(new ValidationException(e.getMessage() + " on row " + get(0), _dataKey.getName())); + return; + } + } + else + typeData.dataIds.add(dataString); + } } // if the data represents a derivation dependency between types, and we're creating ids within the file, diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 11bc9a14ee8..7ff117ba8e9 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -1786,7 +1786,7 @@ public AuditBehaviorType getDefaultAuditBehavior() { var set = new CaseInsensitiveHashSet(); set.addAll(TableInfo.defaultExcludedDetailedUpdateAuditFields); - set.addAll(ExpDataIterators.NOT_FOR_UPDATE); + set.addAll(ExpDataIterators.MATERIAL_NOT_FOR_UPDATE); // We don't want the inventory columns to show up in the sample timeline audit record; // they are captured in their own audit record. set.addAll(InventoryService.InventoryStatusColumn.names()); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 5650e26f871..e6862e32ded 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -1494,8 +1494,8 @@ public DataIterator getDataIterator(DataIteratorContext context) if (!columnNameMap.containsKey(RootMaterialRowId.name())) addAliquotedFrom.addNullColumn(RootMaterialRowId.name(), JdbcType.INTEGER); addAliquotedFrom.addNullColumn(CURRENT_SAMPLE_STATUS_COLUMN_NAME, JdbcType.INTEGER); - addAliquotedFrom.addColumn(new BaseColumnInfo("cpasType", JdbcType.VARCHAR), new SimpleTranslator.ConstantColumn(sampleType.getLSID())); - addAliquotedFrom.addColumn(new BaseColumnInfo("materialSourceId", JdbcType.INTEGER), new SimpleTranslator.ConstantColumn(sampleType.getRowId())); + addAliquotedFrom.addColumn(new BaseColumnInfo(CpasType.fieldKey(), JdbcType.VARCHAR), new SimpleTranslator.ConstantColumn(sampleType.getLSID())); + addAliquotedFrom.addColumn(new BaseColumnInfo(MaterialSourceId.fieldKey(), JdbcType.INTEGER), new SimpleTranslator.ConstantColumn(sampleType.getRowId())); addAliquotedFrom.addNullColumn(ROOT_RECOMPUTE_ROWID_COL, JdbcType.INTEGER); addAliquotedFrom.addNullColumn(PARENT_RECOMPUTE_NAME_COL, JdbcType.VARCHAR); addAliquotedFrom.selectAll(); @@ -1671,14 +1671,14 @@ static class _GenerateNamesDataIterator extends SimpleTranslator else selectAll(CaseInsensitiveHashSet.of(Name.name(), LSID.name(), RootMaterialRowId.name())); - addColumn(new BaseColumnInfo("name", JdbcType.VARCHAR), (Supplier)() -> generatedName); + addColumn(new BaseColumnInfo(Name.fieldKey(), JdbcType.VARCHAR), (Supplier)() -> generatedName); if (!useLsidForUpdate) { DbSequence lsidDbSeq = sampleType.getSampleLsidDbSeq(batchSize, sampleType.getContainer()); - addColumn(new BaseColumnInfo("lsid", JdbcType.VARCHAR), (Supplier) () -> lsidBuilder.setObjectId(String.valueOf(lsidDbSeq.next())).toString()); + addColumn(new BaseColumnInfo(LSID.name(), JdbcType.VARCHAR), (Supplier) () -> lsidBuilder.setObjectId(String.valueOf(lsidDbSeq.next())).toString()); } - addColumn(new BaseColumnInfo("cpasType", JdbcType.VARCHAR), new SimpleTranslator.ConstantColumn(sampleType.getLSID())); - addColumn(new BaseColumnInfo("materialSourceId", JdbcType.INTEGER), new SimpleTranslator.ConstantColumn(sampleType.getRowId())); + addColumn(new BaseColumnInfo(CpasType.fieldKey(), JdbcType.VARCHAR), new SimpleTranslator.ConstantColumn(sampleType.getLSID())); + addColumn(new BaseColumnInfo(MaterialSourceId.fieldKey(), JdbcType.INTEGER), new SimpleTranslator.ConstantColumn(sampleType.getRowId())); } @Override @@ -1882,7 +1882,7 @@ else if (StoredAmount.name().equalsIgnoreCase(name)) { ExpMaterialTable.Column field = entry.getKey(); JdbcType jdbcType = entry.getValue(); - var col = new BaseColumnInfo(field.name(), jdbcType); + var col = new BaseColumnInfo(field.fieldKey(), jdbcType); addColumn(col, new AliquotRollupConvertColumn(field, jdbcType, aliquotedFromDataColInd)); } From 01a03761c88f1cfd72260149e5ec1a751754e066 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 10 Dec 2025 13:14:46 -0800 Subject: [PATCH 55/62] fix --- experiment/src/client/test/integration/SampleTypeCrud.ispec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index f1f2dec9b84..f9ae74339be 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -504,6 +504,7 @@ describe('Import with update / merge', () => { schemaName: 'exp', queryName: 'materials', 'query.columns': 'RowId, Name, MaterialSourceId, Description', + 'query.Name~in': ['FL-1', 'SP-10'].join(';'), 'query.sort': 'RowId', }, { ...topFolderOptions, ...adminOptions }).expect(successfulResponse); From ac820de2aeb83def0b268826c462cef87ac3eaee Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 10 Dec 2025 13:53:35 -0800 Subject: [PATCH 56/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index 10924160f1d..aca9c581024 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" + "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.2.1-fb-remove-sample-lsid.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.1.tgz", - "integrity": "sha512-nmJgZoIKypCKYRrsHqXJPD1krmjO4omqqr1i0ifV813FzfgcVIDH2ogZr7/Oesp6yVPVD195Qfx6fSM/IXcsiA==", + "version": "7.3.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-Drm1DPBkGH+zf3L65dms09oBl69If4QtRv68evUI0uVzcvd6ZVQ8f9Y9RY8w8s5KZHkGKRLWAWsRtf4IiBmzZg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index c06b31a2837..6cb044e4730 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" + "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index 83c8faf57f5..ad0ead3c602 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.1", + "@labkey/components": "7.3.1-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.2.1-fb-remove-sample-lsid.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.1.tgz", - "integrity": "sha512-nmJgZoIKypCKYRrsHqXJPD1krmjO4omqqr1i0ifV813FzfgcVIDH2ogZr7/Oesp6yVPVD195Qfx6fSM/IXcsiA==", + "version": "7.3.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-Drm1DPBkGH+zf3L65dms09oBl69If4QtRv68evUI0uVzcvd6ZVQ8f9Y9RY8w8s5KZHkGKRLWAWsRtf4IiBmzZg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 2df3aa528fb..120dd52d673 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.1", + "@labkey/components": "7.3.1-fb-remove-sample-lsid.0", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index 9e0243f98a1..2ef11cc14c6 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" + "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.2.1-fb-remove-sample-lsid.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.1.tgz", - "integrity": "sha512-nmJgZoIKypCKYRrsHqXJPD1krmjO4omqqr1i0ifV813FzfgcVIDH2ogZr7/Oesp6yVPVD195Qfx6fSM/IXcsiA==", + "version": "7.3.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-Drm1DPBkGH+zf3L65dms09oBl69If4QtRv68evUI0uVzcvd6ZVQ8f9Y9RY8w8s5KZHkGKRLWAWsRtf4IiBmzZg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index f1cd7ae6614..bca10ce32c9 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" + "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index 1ac71c8b765..2ed4421b922 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" + "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.2.1-fb-remove-sample-lsid.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.2.1-fb-remove-sample-lsid.1.tgz", - "integrity": "sha512-nmJgZoIKypCKYRrsHqXJPD1krmjO4omqqr1i0ifV813FzfgcVIDH2ogZr7/Oesp6yVPVD195Qfx6fSM/IXcsiA==", + "version": "7.3.1-fb-remove-sample-lsid.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1-fb-remove-sample-lsid.0.tgz", + "integrity": "sha512-Drm1DPBkGH+zf3L65dms09oBl69If4QtRv68evUI0uVzcvd6ZVQ8f9Y9RY8w8s5KZHkGKRLWAWsRtf4IiBmzZg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index 61f17bfd282..ba8218722d0 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "7.2.1-fb-remove-sample-lsid.1" + "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" }, "devDependencies": { "@labkey/build": "8.7.0", From 92332d97b54cead3e7432f053b0f85a5db40ec3f Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 10 Dec 2025 16:06:56 -0800 Subject: [PATCH 57/62] Use query table only --- .../test/integration/SampleTypeCrud.ispec.ts | 36 ++----------- .../experiment/api/ExpSampleTypeTestCase.jsp | 2 +- .../api/SampleTypeUpdateServiceDI.java | 54 ++++++++----------- 3 files changed, 28 insertions(+), 64 deletions(-) diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index f9ae74339be..75ccf8b70ca 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -469,7 +469,7 @@ describe('Import with update / merge', () => { resp = await importSample(server, tsv, dataType, 'MERGE', topFolderOptions, editorUserOptions); expect(resp.text.indexOf(LSID_MERGE_ERROR) > -1).toBeTruthy(); }); - it('MaterialSourceId is immutable during update', async () => { + it('Cross-type update should not be accepted', async () => { // Arrange const firstSampleType = SAMPLE_ALIQUOT_IMPORT_TYPE_NAME; const secondSampleType = SAMPLE_ALIQUOT_IMPORT_NO_NAME_PATTERN_NAME; @@ -478,43 +478,17 @@ describe('Import with update / merge', () => { const firstRowId = caseInsensitive(firstSample, 'rowId'); const secondRowId = caseInsensitive(secondSample, 'rowId'); - // Fetch sample type rowIds - let resp = await server.post('query', 'selectRows', { - schemaName: 'exp', - queryName: 'SampleSets', - 'query.columns': 'RowId, Name', - 'query.Name~in': [firstSampleType, secondSampleType].join(';'), - 'query.sort': 'Name', - }, { ...topFolderOptions, ...adminOptions }).expect(successfulResponse); - const firstSampleTypeRowId = caseInsensitive(resp.body.rows[0], 'RowId'); - const secondSampleTypeRowId = caseInsensitive(resp.body.rows[1], 'RowId'); - let tsv = 'RowId\tSampleType\tDescription\n'; tsv += `${firstRowId}\t${firstSampleType}\tShould be FL-1\n`; tsv += `${secondRowId}\t${secondSampleType}\tShould be SP-10\n`; // Act - resp = await importSample(server, tsv, firstSampleType, 'UPDATE', topFolderOptions, editorUserOptions); - expect(resp.body.success).toEqual(true); - expect(resp.body.rowCount).toEqual(2); + const resp = await importSample(server, tsv, firstSampleType, 'UPDATE', topFolderOptions, editorUserOptions); // Assert - // Verify that the MaterialSourceId is not altered for these rows - resp = await server.post('query', 'selectRows', { - schemaName: 'exp', - queryName: 'materials', - 'query.columns': 'RowId, Name, MaterialSourceId, Description', - 'query.Name~in': ['FL-1', 'SP-10'].join(';'), - 'query.sort': 'RowId', - }, { ...topFolderOptions, ...adminOptions }).expect(successfulResponse); - - expect(caseInsensitive(resp.body.rows[0], 'Name')).toEqual('FL-1'); - expect(caseInsensitive(resp.body.rows[0], 'MaterialSourceId')).toEqual(firstSampleTypeRowId); - expect(caseInsensitive(resp.body.rows[0], 'Description')).toEqual('Should be FL-1'); - - expect(caseInsensitive(resp.body.rows[1], 'Name')).toEqual('SP-10'); - expect(caseInsensitive(resp.body.rows[1], 'MaterialSourceId')).toEqual(secondSampleTypeRowId); - expect(caseInsensitive(resp.body.rows[1], 'Description')).toEqual('Should be SP-10'); + // Verify that these rows are not updated + expect(resp.body.success).toEqual(false); + expect(resp.body.exception).toContain('Sample does not exist: (RowId)'); }) }); diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index c9c613b8424..b5ef5cde9c8 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -758,7 +758,7 @@ public void testUpdateSomeParents() throws Exception rows.add(CaseInsensitiveHashMap.of("rowId", parent2Type.getSample(c, "P2-1").getRowId(), "MaterialInputs/ChildSamples", "C1")); try { - updateService.updateRows(user, c, rows, null, errors, null, null); + getSampleTypeUpdateService(parent2Type.getName()).updateRows(user, c, rows, null, errors, null, null); fail("Expected to throw exception"); } catch (Exception e) diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index e6862e32ded..70f4ed6732d 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -993,19 +993,31 @@ public boolean hasExistingRowsInOtherContainers(Container container, Map columns, boolean includeParent) {} + private record ExistingRowSelect(Set columns, boolean includeParent) {} private @NotNull ExistingRowSelect getExistingRowSelect(@Nullable Set dataColumns) { if (!(getQueryTable() instanceof UpdateableTableInfo updatable) || dataColumns == null) - return new ExistingRowSelect(getQueryTable(), ALL_COLUMNS, true); + return new ExistingRowSelect(ALL_COLUMNS, true); CaseInsensitiveHashMap remap = updatable.remapSchemaColumns(); if (null == remap) remap = CaseInsensitiveHashMap.of(); // AliquotRollupDataIterator needs "samplestate", "storedamount", "rootmaterialrowId", "units" for MERGE option - Set includedColumns = new CaseInsensitiveHashSet(Name.name(), LSID.name(), RowId.name(), SampleState.name(), StoredAmount.name(), RootMaterialRowId.name(), Units.name()); + // "RawAmount" and "RawUnits" are needed to replace converted amount and unit values with raw values so the + // audit difference is accurate. + Set includedColumns = new CaseInsensitiveHashSet( + LSID.name(), + Name.name(), + RawAmount.name(), + RawUnits.name(), + RootMaterialRowId.name(), + RowId.name(), + SampleState.name(), + StoredAmount.name(), + Units.name() + ); for (ColumnInfo column : getQueryTable().getColumns()) { if (dataColumns.contains(column.getColumnName())) @@ -1014,27 +1026,6 @@ else if (dataColumns.contains(remap.get(column.getColumnName()))) includedColumns.add(remap.get(column.getColumnName())); } - boolean isAllFromMaterialTable = new CaseInsensitiveHashSet(Stream.of(values()) - .map(Enum::name) - .collect(Collectors.toSet())) - .containsAll(includedColumns); - - // only include RawAmount and Raw units if no isAllFromMaterialTable, - // needed to replace converted amount and unit values with raw values so audit difference is accurate - if (!isAllFromMaterialTable) - { - includedColumns.add(RawAmount.name()); - includedColumns.add(RawUnits.name()); - } - - TableInfo selectTable = isAllFromMaterialTable ? ExperimentService.get().getTinfoMaterial() : getQueryTable(); - if (isAllFromMaterialTable) - { - // ExperimentService.get().getTinfoMaterial() uses Container column, not Folder - includedColumns.remove(Folder.name()); - includedColumns.add("container"); - } - boolean hasParentInput = false; if (_sampleType != null) { @@ -1043,7 +1034,7 @@ else if (dataColumns.contains(remap.get(column.getColumnName()))) Map importAliases = _sampleType.getImportAliases(); for (String col : dataColumns) { - if (ExperimentService.isInputOutputColumn(col) || Strings.CI.equals("parent",col) || importAliases.containsKey(col)) + if (ExperimentService.isInputOutputColumn(col) || Strings.CI.equals("parent", col) || importAliases.containsKey(col)) { hasParentInput = true; break; @@ -1055,7 +1046,7 @@ else if (dataColumns.contains(remap.get(column.getColumnName()))) } } - return new ExistingRowSelect(selectTable, includedColumns, hasParentInput); + return new ExistingRowSelect(includedColumns, hasParentInput); } @Override @@ -1069,7 +1060,6 @@ public Map> getExistingRows( ) throws InvalidKeyException, QueryUpdateServiceException { ExistingRowSelect existingRowSelect = getExistingRowSelect(columns); - TableInfo queryTableInfo = existingRowSelect.tableInfo; Set selectColumns = existingRowSelect.columns; Map> sampleRows = new LinkedHashMap<>(); @@ -1106,7 +1096,7 @@ public Map> getExistingRows( missingRowIds = new HashSet<>(rowIdRowNumMap.keySet()); SimpleFilter filter = new SimpleFilter(RowId.fieldKey(), rowIdRowNumMap.keySet(), CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), container); - Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); + Map[] rows = new TableSelector(getQueryTable(), selectColumns, filter, null).getMapArray(); for (Map row : rows) { Long rowId = asLong(row.get(RowId.name())); @@ -1125,7 +1115,7 @@ public Map> getExistingRows( SimpleFilter filter = new SimpleFilter(MaterialSourceId.fieldKey(), sampleTypeId); filter.addCondition(Name.fieldKey(), nameRowNumMap.keySet(), CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), container); - Map[] rows = new TableSelector(queryTableInfo, selectColumns, filter, null).getMapArray(); + Map[] rows = new TableSelector(getQueryTable(), selectColumns, filter, null).getMapArray(); for (Map row : rows) { String name = (String) row.get(Name.name()); @@ -1149,7 +1139,7 @@ public Map> getExistingRows( { SimpleFilter filter = new SimpleFilter(RowId.fieldKey(), missingRowIds, CompareType.IN); filter.addCondition(FieldKey.fromParts("Container"), containerIds, CompareType.IN); - var row = new TableSelector(ExperimentService.get().getTinfoMaterial(), Sets.newCaseInsensitiveHashSet(RowId.name(), Name.name()), filter, null).setMaxRows(1).getMap(); + var row = new TableSelector(ExperimentService.get().getTinfoMaterial(), CaseInsensitiveHashSet.of(RowId.name(), Name.name()), filter, null).setMaxRows(1).getMap(); if (row != null) throw new InvalidKeyException("Sample does not belong to " + container.getName() + " container: " + row.get(Name.name()) + " (" + row.get(RowId.name()) + ")."); } @@ -1160,7 +1150,7 @@ public Map> getExistingRows( filter.addCondition(FieldKey.fromParts("Container"), containerIds, CompareType.IN); filter.addCondition(Name.fieldKey(), missingNames, CompareType.IN); - var row = new TableSelector(ExperimentService.get().getTinfoMaterial(), Sets.newCaseInsensitiveHashSet(Name.name()), filter, null).setMaxRows(1).getMap(); + var row = new TableSelector(ExperimentService.get().getTinfoMaterial(), CaseInsensitiveHashSet.of(Name.name()), filter, null).setMaxRows(1).getMap(); if (row != null) throw new InvalidKeyException("Sample does not belong to " + container.getName() + " container: " + row.get(Name.name()) + "."); } @@ -1176,7 +1166,7 @@ public Map> getExistingRows( } // if contains domain fields, check for aliquot specific fields - if (!queryTableInfo.getName().equalsIgnoreCase("material")) + if (!getQueryTable().getName().equalsIgnoreCase("material")) { Set parentOnlyFields = getSampleMetaFields(); for (Map.Entry> rowNumSampleRow : sampleRows.entrySet()) From 4c41fa12f06b81a9c70f7e10995fcb4f8a7e004c Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Wed, 10 Dec 2025 16:09:14 -0800 Subject: [PATCH 58/62] Allow RowId during merge --- .../test/integration/SampleTypeCrud.ispec.ts | 20 ---------------- .../experiment/api/ExpSampleTypeTestCase.jsp | 23 ------------------- .../api/SampleTypeUpdateServiceDI.java | 5 ---- 3 files changed, 48 deletions(-) diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index 75ccf8b70ca..8ec613962ee 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -426,26 +426,6 @@ describe('Import with update / merge', () => { resp = await importSample(server, updateTsv, dataType, 'UPDATE', topFolderOptions, editorUserOptions); expect(resp.text.indexOf('Sample does not exist') > -1).toBeTruthy(); }); - it('Error when supplying RowId during MERGE', async () => { - const dataType = SAMPLE_ALIQUOT_IMPORT_NO_NAME_PATTERN_NAME; - const sampleName = 'MergeRowIdErrorTest'; - - const rows = await insertRows(server, [{ - name: sampleName, - description: 'created' - }], 'samples', dataType, topFolderOptions, editorUserOptions); - - const rowId = caseInsensitive(rows[0], 'rowId'); - expect(rowId).toBeDefined(); - - // MERGE with RowId should fail - // Even if the name matches and rowId is correct, the presence of the column should trigger the error - const mergeTsv = `RowId\tName\tDescription\n${rowId}\t${sampleName}\tShould fail`; - const resp = await importSample(server, mergeTsv, dataType, 'MERGE', topFolderOptions, editorUserOptions); - - // Check for the specific error message - expect(resp.text.indexOf('RowId is not accepted when merging samples. Specify only the sample name instead.') > -1).toBeTruthy(); - }); it('Error when supplying LSID without RowId or Name', async () => { const dataType = SAMPLE_ALIQUOT_IMPORT_NO_NAME_PATTERN_NAME; const sampleName = 'LsidKeyErrorTest'; diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index b5ef5cde9c8..0e427239df2 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -70,7 +70,6 @@ <%@ page import="java.io.StringBufferInputStream" %> <%@ page import="java.util.ArrayList" %> <%@ page import="java.util.Arrays" %> -<%@ page import="java.util.Collection" %> <%@ page import="static org.hamcrest.CoreMatchers.containsString" %> <%@ page import="java.util.Collections" %> <%@ page import="java.util.HashMap" %> @@ -783,28 +782,6 @@ public void testUpdateSomeParents() throws Exception opts.setParents(true); opts.setDepth(2); - // Attempt to merge using rowIds - { - rows.clear(); - rows.add(CaseInsensitiveHashMap.of("rowId", C1.getRowId(), "name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); - rows.add(CaseInsensitiveHashMap.of("rowId", C4.getRowId(), "name", "C5", "MaterialInputs/Parent1Samples", null)); // intentionally mix up name - - updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); - assertThat(errors.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); - errors = new BatchValidationException(); - } - - // Attempt to merge using "Row Id" label - { - rows.clear(); - rows.add(CaseInsensitiveHashMap.of("Row Id", C1.getRowId(), "name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); - rows.add(CaseInsensitiveHashMap.of("Row Id", C4.getRowId(), "name", "C5", "MaterialInputs/Parent1Samples", null)); // intentionally mix up name - - updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); - assertThat(errors.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); - errors = new BatchValidationException(); - } - // Attempt to update using outdated "LSID" and do not specify any other keys // Note: using try/catch here as updateRows() executes with retry which throws // if validation exceptions are encountered. diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 70f4ed6732d..825da37ca58 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -1450,11 +1450,6 @@ public DataIterator getDataIterator(DataIteratorContext context) keysCheck.add(RowId.name()); if (isUpdate) continue; - if (isMerge) - { - context.getErrors().addRowError(new ValidationException("RowId is not accepted when merging samples. Specify only the sample name instead.", RowId.name())); - return null; - } } if (isExpMaterialColumn(LSID, name)) keysCheck.add(LSID.name()); From 2f3da9481168f23a789c5bcc1d35266f3d6daa58 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Thu, 11 Dec 2025 16:46:17 -0800 Subject: [PATCH 59/62] ExpMaterialImpl: set MaterialSourceId on insert --- .../org/labkey/experiment/api/ExpMaterialImpl.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java index 9aadd3756a7..7108fc43a05 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialImpl.java @@ -343,6 +343,18 @@ protected void save(User user, TableInfo table, boolean ensureObject) setRowId((int) longId); if (null == getRootMaterialRowId()) setRootMaterialRowId(getRowId()); + + // If a MaterialSourceId is not yet specified and the material is associated with a sample type, + // then set the MaterialSourceId to the sample type + if (null == _object.getMaterialSourceId()) + { + ExpSampleType st = getSampleType(); + if (st != null) + { + assert st.getLSID().equals(getCpasType()); + _object.setMaterialSourceId(st.getRowId()); + } + } } super.save(user, table, true, isInsert); } From ae75e1aa53988c3be3c4ed07ed6aed0fd6350f70 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 12 Dec 2025 11:07:56 -0800 Subject: [PATCH 60/62] Summary audit event, DataIteratorPartitions --- .../api/audit/TransactionAuditProvider.java | 5 ++- .../api/query/AbstractQueryUpdateService.java | 38 ++++++++++--------- .../labkey/api/query/QueryUpdateService.java | 3 +- .../api/SampleTypeUpdateServiceDI.java | 16 +++++++- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/api/src/org/labkey/api/audit/TransactionAuditProvider.java b/api/src/org/labkey/api/audit/TransactionAuditProvider.java index 7555a2dd87f..4e2c1ff9c50 100644 --- a/api/src/org/labkey/api/audit/TransactionAuditProvider.java +++ b/api/src/org/labkey/api/audit/TransactionAuditProvider.java @@ -133,6 +133,7 @@ public enum TransactionDetail Action(false, "The controller-action for this request"), AuditEvents(true, "The types of audit events generated during the transaction"), ClientLibrary(false, "The client library (R, Python, etc) used to perform the action"), + DataIteratorPartitions(false, "The number of partitions rows were processed in via data iterator"), DataIteratorUsed(false, "If data iterator was used for insert/update"), EditMethod(false, "The method used to insert/update data from the app (e.g., 'DetailEdit', 'GridEdit', etc)"), ETL(true, "The ETL process name involved in the transaction"), @@ -159,7 +160,7 @@ public static TransactionDetail fromString(String key) return null; } - public static void addAuditDetails(@NotNull Map transactionDetails, @NotNull Map auditDetails) + public static void addAuditDetails(@NotNull Map transactionDetails, @NotNull Map auditDetails) { if (!auditDetails.isEmpty()) { @@ -172,7 +173,7 @@ public static void addAuditDetails(@NotNull Map transactionDetails, @NotNull String auditDetailsJson) + public static void addAuditDetails(@NotNull Map transactionDetails, @NotNull String auditDetailsJson) { if (StringUtils.isEmpty(auditDetailsJson)) return; diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index b90de00d09b..1b64835bb42 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -431,22 +431,11 @@ protected int _importRowsUsingDIB(User user, Container container, DataIteratorBu if (context.getErrors().hasErrors()) return 0; - else - { - if (!context.isCrossTypeImport() && !context.isCrossFolderImport()) // audit handled at table level - { - AuditBehaviorType auditType = (AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior); - String auditUserComment = (String) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment); - boolean skipAuditLevelCheck = false; - if (context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad)) - { - if (getQueryTable().getEffectiveAuditBehavior(auditType) == AuditBehaviorType.DETAILED) // allow ETL to demote audit level for bulkLoad - skipAuditLevelCheck = true; - } - getQueryTable().getAuditHandler(auditType).addSummaryAuditEvent(user, container, getQueryTable(), context.getInsertOption().auditAction, count, auditType, auditUserComment, skipAuditLevelCheck); - } - return count; - } + + if (!context.getConfigParameterBoolean(ConfigParameters.SkipAuditSummary)) + _addSummaryAuditEvent(container, user, context, count); + + return count; } protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) @@ -459,7 +448,6 @@ protected DataIteratorBuilder postTriggerDataIterator(DataIteratorBuilder out, D return out; } - /** this is extracted so subclasses can add wrap */ protected int _pump(DataIteratorBuilder etl, final @Nullable ArrayList> rows, DataIteratorContext context) { @@ -1164,6 +1152,22 @@ static FileLike checkFileUnderRoot(Container container, FileLike file) throws Ex return file; } + protected void _addSummaryAuditEvent(Container container, User user, DataIteratorContext context, int count) + { + if (!context.isCrossTypeImport() && !context.isCrossFolderImport()) // audit handled at table level + { + AuditBehaviorType auditType = (AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior); + String auditUserComment = (String) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment); + boolean skipAuditLevelCheck = false; + if (context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad)) + { + if (getQueryTable().getEffectiveAuditBehavior(auditType) == AuditBehaviorType.DETAILED) // allow ETL to demote audit level for bulkLoad + skipAuditLevelCheck = true; + } + getQueryTable().getAuditHandler(auditType).addSummaryAuditEvent(user, container, getQueryTable(), context.getInsertOption().auditAction, count, auditType, auditUserComment, skipAuditLevelCheck); + } + } + /** * Is used by the AttachmentDataIterator to point to the location of the serialized * attachment files. diff --git a/api/src/org/labkey/api/query/QueryUpdateService.java b/api/src/org/labkey/api/query/QueryUpdateService.java index 53925404dec..09c9e4f6495 100644 --- a/api/src/org/labkey/api/query/QueryUpdateService.java +++ b/api/src/org/labkey/api/query/QueryUpdateService.java @@ -114,7 +114,8 @@ enum ConfigParameters PreferPKOverObjectUriAsKey, // (Bool) Prefer getPkColumnNames instead of getObjectURIColumnName to use as keys SkipReselectRows, // (Bool) If true, skip qus.getRows and use raw returned rows. Applicable for CommandType.insert/insertWithKeys/update/updateChangingKeys TargetContainer, - ByPassAudit // (Bool) If true, skip DetailedAuditLogDataIterator. For internal use only, don't expose for API. + ByPassAudit, // (Bool) If true, skip DetailedAuditLogDataIterator. For internal use only, don't expose for API. + SkipAuditSummary, // (Bool) If true, skip audit summary logging. For internal use only, don't expose for API.' } diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 825da37ca58..8517e84e9f8 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -23,6 +23,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.LongHashSet; @@ -124,7 +125,6 @@ import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static org.labkey.api.audit.AuditHandler.DELTA_PROVIDED_DATA_PREFIX; @@ -532,6 +532,7 @@ public List> updateRows( results = getSchema().getDbSchema().getScope().executeWithRetry(tx -> { int index = 0; + int numPartitions = 0; List> ret = new ArrayList<>(); while (index < rows.size()) @@ -545,9 +546,13 @@ public List> updateRows( List> rowsToProcess = rows.subList(index, nextIndex); index = nextIndex; + numPartitions++; DataIteratorContext context = getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters); + // skip audit summary for the partitions, we will perform it once at the end + context.putConfigParameter(ConfigParameters.SkipAuditSummary, true); + List> subRet = super._updateRowsUsingDIB(user, container, rowsToProcess, context, extraScriptContext); // we need to throw if we don't want executeWithRetry() attempt commit() @@ -558,6 +563,15 @@ public List> updateRows( ret.addAll(subRet); } + if (numPartitions > 1) + { + var auditEvent = tx.getAuditEvent(); + if (auditEvent != null) + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.DataIteratorPartitions, numPartitions); + } + + _addSummaryAuditEvent(container, user, getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters), ret.size()); + return ret; }); } From 814697648952e5105c6bc43dd9969ebf904ec9ef Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Fri, 12 Dec 2025 14:42:27 -0800 Subject: [PATCH 61/62] Experimental feature, check for duplicates --- .../test/integration/SampleTypeCrud.ispec.ts | 21 ++++++++- .../labkey/experiment/ExperimentModule.java | 3 ++ .../experiment/api/ExpSampleTypeTestCase.jsp | 22 +++++++++ .../api/SampleTypeUpdateServiceDI.java | 47 +++++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) diff --git a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts index 8ec613962ee..bb71be1260a 100644 --- a/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts +++ b/experiment/src/client/test/integration/SampleTypeCrud.ispec.ts @@ -12,7 +12,6 @@ import { import { caseInsensitive, SAMPLE_TYPE_DESIGNER_ROLE } from '@labkey/components'; const { importSample, insertRows } = ExperimentCRUDUtils; -// @ts-expect-error process is not available in a browser environment const server = hookServer(process.env); const PROJECT_NAME = 'SampleTypeCrudJestProject'; @@ -426,6 +425,26 @@ describe('Import with update / merge', () => { resp = await importSample(server, updateTsv, dataType, 'UPDATE', topFolderOptions, editorUserOptions); expect(resp.text.indexOf('Sample does not exist') > -1).toBeTruthy(); }); + it('Error when supplying RowId during MERGE', async () => { + const dataType = SAMPLE_ALIQUOT_IMPORT_NO_NAME_PATTERN_NAME; + const sampleName = 'MergeRowIdErrorTest'; + + const rows = await insertRows(server, [{ + name: sampleName, + description: 'created' + }], 'samples', dataType, topFolderOptions, editorUserOptions); + + const rowId = caseInsensitive(rows[0], 'rowId'); + expect(rowId).toBeDefined(); + + // MERGE with RowId should fail + // Even if the name matches and rowId is correct, the presence of the column should trigger the error + const mergeTsv = `RowId\tName\tDescription\n${rowId}\t${sampleName}\tShould fail`; + const resp = await importSample(server, mergeTsv, dataType, 'MERGE', topFolderOptions, editorUserOptions); + + // Check for the specific error message + expect(resp.text.indexOf('RowId is not accepted when merging samples. Specify only the sample name instead.') > -1).toBeTruthy(); + }); it('Error when supplying LSID without RowId or Name', async () => { const dataType = SAMPLE_ALIQUOT_IMPORT_NO_NAME_PATTERN_NAME; const sampleName = 'LsidKeyErrorTest'; diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 7ccbc8684d1..dd9c3c52d8e 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -129,6 +129,7 @@ import org.labkey.experiment.api.LogDataType; import org.labkey.experiment.api.Protocol; import org.labkey.experiment.api.SampleTypeServiceImpl; +import org.labkey.experiment.api.SampleTypeUpdateServiceDI; import org.labkey.experiment.api.UniqueValueCounterTestCase; import org.labkey.experiment.api.VocabularyDomainKind; import org.labkey.experiment.api.data.ChildOfCompareType; @@ -272,6 +273,8 @@ protected void init() "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", "Support for querying lineage of experiment objects", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(SampleTypeUpdateServiceDI.EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE, "Allow RowId to be accepted when merging samples", + "If the incoming data includes a RowId column we will allow the column but ignore it's values.", false); RoleManager.registerPermission(new DesignVocabularyPermission(), true); RoleManager.registerRole(new SampleTypeDesignerRole()); diff --git a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp index 0e427239df2..10e13d4a0ea 100644 --- a/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp +++ b/experiment/src/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp @@ -782,6 +782,28 @@ public void testUpdateSomeParents() throws Exception opts.setParents(true); opts.setDepth(2); + // Attempt to merge using rowIds + { + rows.clear(); + rows.add(CaseInsensitiveHashMap.of("rowId", C1.getRowId(), "name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); + rows.add(CaseInsensitiveHashMap.of("rowId", C4.getRowId(), "name", "C5", "MaterialInputs/Parent1Samples", null)); // intentionally mix up name + + updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); + assertThat(errors.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); + errors = new BatchValidationException(); + } + + // Attempt to merge using "Row Id" label + { + rows.clear(); + rows.add(CaseInsensitiveHashMap.of("Row Id", C1.getRowId(), "name", "C1", "MaterialInputs/Parent1Samples", "P1-1")); + rows.add(CaseInsensitiveHashMap.of("Row Id", C4.getRowId(), "name", "C5", "MaterialInputs/Parent1Samples", null)); // intentionally mix up name + + updateService.mergeRows(user, c, MapDataIterator.of(rows), errors, null, null); + assertThat(errors.getMessage(), containsString("RowId is not accepted when merging samples. Specify only the sample name instead.")); + errors = new BatchValidationException(); + } + // Attempt to update using outdated "LSID" and do not specify any other keys // Note: using try/catch here as updateRows() executes with retry which throws // if validation exceptions are encountered. diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 8517e84e9f8..1db58f8c260 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -99,6 +99,7 @@ import org.labkey.api.security.User; import org.labkey.api.security.permissions.MoveEntitiesPermission; import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.study.publish.StudyPublishService; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.GUID; @@ -160,6 +161,8 @@ public class SampleTypeUpdateServiceDI extends DefaultQueryUpdateService public static final String ROOT_RECOMPUTE_ROWID_SET = "RootIdToRecomputeSet"; public static final String PARENT_RECOMPUTE_NAME_SET = "ParentNameToRecomputeSet"; + public static final String EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE = "org.labkey.experiment.api.SampleTypeUpdateServiceDI#ALLOW_ROW_ID_SAMPLE_MERGE"; + public static final Map SAMPLE_ALT_IMPORT_NAME_COLS; private static final Map ALIQUOT_ROLLUP_FIELDS = Map.of( @@ -535,6 +538,9 @@ public List> updateRows( int numPartitions = 0; List> ret = new ArrayList<>(); + Set observedRowIds = new HashSet<>(); + Set observedNames = new CaseInsensitiveHashSet(); + while (index < rows.size()) { CaseInsensitiveHashSet rowKeys = new CaseInsensitiveHashSet(rows.get(index).keySet()); @@ -560,7 +566,20 @@ public List> updateRows( throw new DbScope.RetryPassthroughException(context.getErrors()); if (subRet != null) + { ret.addAll(subRet); + + // Check if duplicate rows have been processed across the partitions + // Only start checking for duplicates after the first partition has been processed. + if (numPartitions > 1) + { + // If we are on the second partition, then lazily check all previous rows, otherwise check only the current partition + checkPartitionForDuplicates(numPartitions == 2 ? ret : subRet, observedRowIds, observedNames, errors); + } + + if (errors.hasErrors()) + throw new DbScope.RetryPassthroughException(errors); + } } if (numPartitions > 1) @@ -590,6 +609,26 @@ public List> updateRows( return results; } + private void checkPartitionForDuplicates(List> partitionRows, Set globalRowIds, Set globalNames, BatchValidationException errors) + { + for (Map row : partitionRows) + { + Long rowId = MapUtils.getLong(row, RowId.name()); + if (rowId != null && !globalRowIds.add(rowId)) + { + errors.addRowError(new ValidationException("Duplicate key provided: " + rowId)); + return; + } + + Object nameObj = row.get(Name.name()); + if (nameObj != null && !globalNames.add(nameObj.toString())) + { + errors.addRowError(new ValidationException("Duplicate key provided: " + nameObj)); + return; + } + } + } + /** * Attempt to make the passed in types match the expected types so the script doesn't have to do the conversion */ @@ -1464,6 +1503,14 @@ public DataIterator getDataIterator(DataIteratorContext context) keysCheck.add(RowId.name()); if (isUpdate) continue; + + // While accepting RowId during merge is not our preferred behavior, we want to give users a way + // to opt-in to the old behavior where RowId is accepted and ignored. + if (isMerge && !OptionalFeatureService.get().isFeatureEnabled(EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE)) + { + context.getErrors().addRowError(new ValidationException("RowId is not accepted when merging samples. Specify only the sample name instead.", RowId.name())); + return null; + } } if (isExpMaterialColumn(LSID, name)) keysCheck.add(LSID.name()); From b43d54712715240bbacb07cb2190412753108739 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Sat, 13 Dec 2025 10:46:07 -0800 Subject: [PATCH 62/62] Bump @labkey packages --- assay/package-lock.json | 8 ++++---- assay/package.json | 2 +- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- pipeline/package-lock.json | 8 ++++---- pipeline/package.json | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/assay/package-lock.json b/assay/package-lock.json index aca9c581024..a5783fdc908 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.3.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2525,9 +2525,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.3.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-Drm1DPBkGH+zf3L65dms09oBl69If4QtRv68evUI0uVzcvd6ZVQ8f9Y9RY8w8s5KZHkGKRLWAWsRtf4IiBmzZg==", + "version": "7.3.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1.tgz", + "integrity": "sha512-rGH7uNiU3yWMhIUbcQZtctom8dbfrFFzhrHhcOcqzSNEQ6mH1/XXOPGmNdSsgAwKidqdgDfkTd5y5QC+I7PBmA==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/assay/package.json b/assay/package.json index 6cb044e4730..e0cddb3cbe4 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.3.1" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index ad0ead3c602..763936d194f 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.3.1-fb-remove-sample-lsid.0", + "@labkey/components": "7.3.1", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.3.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-Drm1DPBkGH+zf3L65dms09oBl69If4QtRv68evUI0uVzcvd6ZVQ8f9Y9RY8w8s5KZHkGKRLWAWsRtf4IiBmzZg==", + "version": "7.3.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1.tgz", + "integrity": "sha512-rGH7uNiU3yWMhIUbcQZtctom8dbfrFFzhrHhcOcqzSNEQ6mH1/XXOPGmNdSsgAwKidqdgDfkTd5y5QC+I7PBmA==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 120dd52d673..31f80eaf0d3 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.3.1-fb-remove-sample-lsid.0", + "@labkey/components": "7.3.1", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index 2ef11cc14c6..d189bfe57f7 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.3.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.3.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-Drm1DPBkGH+zf3L65dms09oBl69If4QtRv68evUI0uVzcvd6ZVQ8f9Y9RY8w8s5KZHkGKRLWAWsRtf4IiBmzZg==", + "version": "7.3.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1.tgz", + "integrity": "sha512-rGH7uNiU3yWMhIUbcQZtctom8dbfrFFzhrHhcOcqzSNEQ6mH1/XXOPGmNdSsgAwKidqdgDfkTd5y5QC+I7PBmA==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index bca10ce32c9..b6fe58b8519 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.3.1" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index 2ed4421b922..4b1397a8fbc 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.3.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2759,9 +2759,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.3.1-fb-remove-sample-lsid.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1-fb-remove-sample-lsid.0.tgz", - "integrity": "sha512-Drm1DPBkGH+zf3L65dms09oBl69If4QtRv68evUI0uVzcvd6ZVQ8f9Y9RY8w8s5KZHkGKRLWAWsRtf4IiBmzZg==", + "version": "7.3.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.3.1.tgz", + "integrity": "sha512-rGH7uNiU3yWMhIUbcQZtctom8dbfrFFzhrHhcOcqzSNEQ6mH1/XXOPGmNdSsgAwKidqdgDfkTd5y5QC+I7PBmA==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/pipeline/package.json b/pipeline/package.json index ba8218722d0..a62353c2f69 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "7.3.1-fb-remove-sample-lsid.0" + "@labkey/components": "7.3.1" }, "devDependencies": { "@labkey/build": "8.7.0",