Skip to content

Commit ee6f7c3

Browse files
labkey-klumlabkey-klum
committed
Don't save bulk import data or offer a download link
1 parent 2893e15 commit ee6f7c3

File tree

7 files changed

+58
-135
lines changed

7 files changed

+58
-135
lines changed

‎study/src/org/labkey/study/StudyModule.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,6 @@
4242
import org.labkey.api.exp.PropertyType;
4343
import org.labkey.api.exp.api.ExperimentService;
4444
import org.labkey.api.exp.property.PropertyService;
45-
import org.labkey.api.files.FileContentService;
46-
import org.labkey.api.files.TableUpdaterFileListener;
4745
import org.labkey.api.message.digest.ReportAndDatasetChangeDigestProvider;
4846
import org.labkey.api.migration.DatabaseMigrationService;
4947
import org.labkey.api.migration.DefaultMigrationSchemaHandler;
@@ -228,7 +226,7 @@ public String getName()
228226
@Override
229227
public Double getSchemaVersion()
230228
{
231-
return 25.003;
229+
return 25.004;
232230
}
233231

234232
@Override
@@ -391,8 +389,6 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext)
391389
folderRegistry.addFactories(new StudyWriterFactory(), new StudyImporterFactory());
392390
}
393391

394-
FileContentService.get().addFileListener(new TableUpdaterFileListener(StudySchema.getInstance().getTableInfoUploadLog(), "FilePath", TableUpdaterFileListener.Type.filePath, "RowId"));
395-
396392
DatasetDefinition.cleanupOrphanedDatasetDomains();
397393

398394
OptionalFeatureService.get().addExperimentalFeatureFlag(StudyQuerySchema.EXPERIMENTAL_STUDY_SUBSCHEMAS, "Use sub-schemas in Study",

‎study/src/org/labkey/study/assay/StudyPublishManager.java‎

Lines changed: 15 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,11 @@
1818

1919
import org.apache.commons.beanutils.ConvertUtils;
2020
import org.apache.commons.collections4.MapUtils;
21-
import org.apache.commons.io.IOUtils;
2221
import org.apache.logging.log4j.LogManager;
2322
import org.apache.logging.log4j.Logger;
2423
import org.jetbrains.annotations.NotNull;
2524
import org.jetbrains.annotations.Nullable;
2625
import org.labkey.api.assay.AbstractAssayProvider;
27-
import org.labkey.api.assay.AssayFileWriter;
2826
import org.labkey.api.assay.AssayProtocolSchema;
2927
import org.labkey.api.assay.AssayProvider;
3028
import org.labkey.api.assay.AssayService;
@@ -113,9 +111,7 @@
113111
import org.labkey.api.study.publish.StudyDatasetLinkedColumn;
114112
import org.labkey.api.study.publish.StudyPublishService;
115113
import org.labkey.api.study.query.PublishResultsQueryView;
116-
import org.labkey.api.util.DateUtil;
117114
import org.labkey.api.util.FileStream;
118-
import org.labkey.api.util.FileUtil;
119115
import org.labkey.api.util.PageFlowUtil;
120116
import org.labkey.api.util.Pair;
121117
import org.labkey.api.util.StringExpressionFactory;
@@ -140,10 +136,7 @@
140136

141137
import java.io.Closeable;
142138
import java.io.IOException;
143-
import java.io.OutputStream;
144139
import java.math.BigDecimal;
145-
import java.nio.file.Files;
146-
import java.nio.file.Path;
147140
import java.util.ArrayList;
148141
import java.util.Arrays;
149142
import java.util.Collection;
@@ -932,60 +925,10 @@ private static String createUniqueDatasetName(Study study, String assayName)
932925
return name;
933926
}
934927

935-
public UploadLog saveUploadData(User user, Dataset dsd, FileStream tsv, String filename) throws IOException
936-
{
937-
PipeRoot pipelineRoot = PipelineService.get().findPipelineRoot(dsd.getContainer());
938-
if (null == pipelineRoot || !pipelineRoot.isValid())
939-
throw new IOException("Please have your administrator set up a pipeline root for this folder.");
940-
941-
Path dir = pipelineRoot.resolveToNioPath(AssayFileWriter.DIR_NAME);
942-
if (null == dir)
943-
throw new IOException("Cannot create directory uploaded data: " + AssayFileWriter.DIR_NAME);
944-
945-
if (!Files.exists(dir))
946-
{
947-
FileUtil.createDirectory(dir);
948-
}
949-
950-
//File name is studyname_datasetname_date_hhmm.ss
951-
Date dateCreated = new Date();
952-
String dateString = DateUtil.formatDateTime(dateCreated, "yyy-MM-dd-HHmm");
953-
int id = 0;
954-
Path file;
955-
do
956-
{
957-
String extension = Objects.toString(filename == null ? "tsv" : FileUtil.getExtension(filename), "tsv");
958-
String extra = id++ == 0 ? "" : String.valueOf(id);
959-
String fileName = dsd.getStudy().getLabel() + "-" + dsd.getLabel() + "-" + dateString + extra + "." + extension;
960-
fileName = fileName.replace('\\', '_').replace('/', '_').replace(':', '_');
961-
file = FileUtil.appendName(dir, fileName);
962-
}
963-
while (Files.exists(file));
964-
965-
try (OutputStream out = Files.newOutputStream(file))
966-
{
967-
IOUtils.copy(tsv.openInputStream(), out);
968-
tsv.closeInputStream();
969-
}
970-
971-
UploadLog ul = new UploadLog();
972-
ul.setContainer(dsd.getContainer());
973-
ul.setDatasetId(dsd.getDatasetId());
974-
ul.setCreated(dateCreated);
975-
ul.setUserId(user.getUserId());
976-
ul.setStatus("Initializing");
977-
String filePath = FileUtil.hasCloudScheme(file) ? FileUtil.pathToString(file) : file.toFile().getPath();
978-
ul.setFilePath(filePath);
979-
980-
return Table.insert(user, getTinfoUpdateLog(), ul);
981-
}
982-
983-
984928
/**
985-
* Return an array of LSIDs from the newly created dataset entries,
986-
* along with the upload log.
929+
* Return an array of LSIDs from the newly created dataset entries.
987930
*/
988-
public Pair<List<String>, UploadLog> importDatasetTSV(User user, StudyImpl study, DatasetDefinition dsd, DataLoader dl, LookupResolutionType lookupResolutionType, FileStream fileIn, String originalFileName, Map<String, String> columnMap, BatchValidationException errors, QueryUpdateService.InsertOption insertOption, @Nullable AuditBehaviorType auditBehaviorType)
931+
public List<String> importDatasetTSV(User user, StudyImpl study, DatasetDefinition dsd, DataLoader dl, LookupResolutionType lookupResolutionType, FileStream fileIn, String originalFileName, Map<String, String> columnMap, BatchValidationException errors, QueryUpdateService.InsertOption insertOption, @Nullable AuditBehaviorType auditBehaviorType)
989932
{
990933
DbScope scope = StudySchema.getInstance().getScope();
991934

@@ -995,7 +938,13 @@ public Pair<List<String>, UploadLog> importDatasetTSV(User user, StudyImpl study
995938
try
996939
{
997940
if (null != fileIn)
998-
ul = saveUploadData(user, dsd, fileIn, originalFileName);
941+
{
942+
ul = new UploadLog();
943+
ul.setContainer(dsd.getContainer());
944+
ul.setDatasetId(dsd.getDatasetId());
945+
ul.setUserId(user.getUserId());
946+
ul.setStatus("Initializing");
947+
}
999948

1000949
try (DbScope.Transaction transaction = scope.ensureTransaction())
1001950
{
@@ -1025,8 +974,9 @@ public Pair<List<String>, UploadLog> importDatasetTSV(User user, StudyImpl study
1025974
ul.setStatus("ERROR");
1026975
String description = ul.getDescription();
1027976
ul.setDescription(description == null ? "" : description + "\n" + new Date() + ":" + x.getMessage());
1028-
ul = Table.update(user, StudySchema.getInstance().getTableInfoUploadLog(), ul, ul.getRowId());
1029-
return Pair.of(lsids, ul);
977+
Table.insert(user, getTinfoUpdateLog(), ul);
978+
979+
return lsids;
1030980
}
1031981
}
1032982

@@ -1035,7 +985,7 @@ public Pair<List<String>, UploadLog> importDatasetTSV(User user, StudyImpl study
1035985
//Update the status
1036986
assert ul != null : "Upload log should always exist if no errors have occurred.";
1037987
ul.setStatus("SUCCESS");
1038-
ul = Table.update(user, getTinfoUpdateLog(), ul, ul.getRowId());
988+
Table.insert(user, getTinfoUpdateLog(), ul);
1039989
}
1040990
else if (ul != null)
1041991
{
@@ -1048,17 +998,9 @@ else if (ul != null)
1048998
sep = "\n";
1049999
}
10501000
ul.setDescription(sb.toString());
1051-
ul = Table.update(user, getTinfoUpdateLog(), ul, ul.getRowId());
1001+
Table.insert(user, getTinfoUpdateLog(), ul);
10521002
}
1053-
return Pair.of(lsids, ul);
1054-
}
1055-
1056-
public UploadLog getUploadLog(Container c, int id)
1057-
{
1058-
SimpleFilter filter = SimpleFilter.createContainerFilter(c);
1059-
filter.addCondition(FieldKey.fromParts("rowId"), id);
1060-
1061-
return new TableSelector(getTinfoUpdateLog(), filter, null).getObject(UploadLog.class);
1003+
return lsids;
10621004
}
10631005

10641006
@Override

‎study/src/org/labkey/study/controllers/StudyController.java‎

Lines changed: 5 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,6 @@
9090
import org.labkey.api.data.RuntimeSQLException;
9191
import org.labkey.api.data.SQLFragment;
9292
import org.labkey.api.data.ShowRows;
93-
import org.labkey.api.data.SimpleDisplayColumn;
9493
import org.labkey.api.data.SimpleFilter;
9594
import org.labkey.api.data.Sort;
9695
import org.labkey.api.data.SqlExecutor;
@@ -227,7 +226,6 @@
227226
import org.labkey.api.view.template.EmptyView;
228227
import org.labkey.api.view.template.PageConfig;
229228
import org.labkey.api.writer.FileSystemFile;
230-
import org.labkey.api.writer.HtmlWriter;
231229
import org.labkey.api.writer.VirtualFile;
232230
import org.labkey.data.xml.TablesDocument;
233231
import org.labkey.study.CohortFilterFactory;
@@ -2712,19 +2710,19 @@ protected int importData(DataLoader dl, FileStream file, String originalName, Ba
27122710
columnMap.put(_form.getSequenceNum(), column);
27132711
}
27142712

2715-
Pair<List<String>, UploadLog> result = StudyPublishManager.getInstance().importDatasetTSV(getUser(), _study, _def, dl, getLookupResolutionType(), file, originalName, columnMap, errors, _form.getInsertOption(), auditBehaviorType);
2713+
List<String> lsids = StudyPublishManager.getInstance().importDatasetTSV(getUser(), _study, _def, dl, getLookupResolutionType(), file, originalName, columnMap, errors, _form.getInsertOption(), auditBehaviorType);
27162714

2717-
if (!result.getKey().isEmpty())
2715+
if (!lsids.isEmpty())
27182716
{
27192717
// Log the import when SUMMARY is configured, if DETAILED is configured the DetailedAuditLogDataIterator will handle each row change.
27202718
// It would be nice in the future to replace the DetailedAuditLogDataIterator with a general purpose AuditLogDataIterator
27212719
// that can delegate the audit behavior type to the AuditDataHandler, so this code can go away
27222720
//
2723-
String comment = "Dataset data imported. " + result.getKey().size() + " rows imported";
2724-
new DatasetDefinition.DatasetAuditHandler(_def).addAuditEvent(getUser(), getContainer(), AuditBehaviorType.SUMMARY, comment, result.getValue());
2721+
String comment = "Dataset data imported. " + lsids.size() + " rows imported";
2722+
new DatasetDefinition.DatasetAuditHandler(_def).addAuditEvent(getUser(), getContainer(), AuditBehaviorType.SUMMARY, comment);
27252723
}
27262724

2727-
return result.getKey().size();
2725+
return lsids.size();
27282726
}
27292727

27302728
@Override
@@ -2855,15 +2853,6 @@ public ModelAndView getView(IdForm form, BindException errors)
28552853
DataRegion dr = new DataRegion();
28562854
dr.addColumns(tInfo, "RowId,Created,CreatedBy,Status,Description");
28572855
GridView gv = new GridView(dr, errors);
2858-
DisplayColumn dc = new SimpleDisplayColumn(null) {
2859-
@Override
2860-
public void renderGridCellContents(RenderContext ctx, HtmlWriter out)
2861-
{
2862-
ActionURL url = new ActionURL(DownloadTsvAction.class, ctx.getContainer()).addParameter("id", String.valueOf(ctx.get("RowId")));
2863-
out.write(LinkBuilder.labkeyLink("Download Data File", url));
2864-
}
2865-
};
2866-
dr.addDisplayColumn(dc);
28672856

28682857
SimpleFilter filter = SimpleFilter.createContainerFilter(getContainer());
28692858
if (form.getId() != 0)
@@ -2885,24 +2874,6 @@ public void addNavTrail(NavTree root)
28852874
}
28862875
}
28872876

2888-
@RequiresPermission(UpdatePermission.class)
2889-
public static class DownloadTsvAction extends SimpleViewAction<IdForm>
2890-
{
2891-
@Override
2892-
public ModelAndView getView(IdForm form, BindException errors) throws Exception
2893-
{
2894-
UploadLog ul = StudyPublishManager.getInstance().getUploadLog(getContainer(), form.getId());
2895-
PageFlowUtil.streamFile(getViewContext().getResponse(), new File(ul.getFilePath()).toPath(), true);
2896-
2897-
return null;
2898-
}
2899-
2900-
@Override
2901-
public void addNavTrail(NavTree root)
2902-
{
2903-
}
2904-
}
2905-
29062877
@RequiresPermission(ReadPermission.class)
29072878
public static class DatasetItemDetailsAction extends SimpleViewAction<SourceLsidForm>
29082879
{

‎study/src/org/labkey/study/dataset/DatasetSnapshotProvider.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -487,7 +487,7 @@ public synchronized ActionURL updateSnapshot(QuerySnapshotForm form, BindExcepti
487487

488488
ViewContext context = form.getViewContext();
489489
new DatasetDefinition.DatasetAuditHandler(dsDef).addAuditEvent(context.getUser(), context.getContainer(), AuditBehaviorType.DETAILED,
490-
"Dataset snapshot was updated. " + numRowsDeleted + " rows were removed and replaced with " + newRows.size() + " rows.", null);
490+
"Dataset snapshot was updated. " + numRowsDeleted + " rows were removed and replaced with " + newRows.size() + " rows.");
491491

492492
def.setLastUpdated(new Date());
493493
def.save(form.getViewContext().getUser());
@@ -504,7 +504,7 @@ public synchronized ActionURL updateSnapshot(QuerySnapshotForm form, BindExcepti
504504
{
505505
ViewContext context = form.getViewContext();
506506
new DatasetDefinition.DatasetAuditHandler(dsDef).addAuditEvent(context.getUser(), context.getContainer(), AuditBehaviorType.DETAILED,
507-
"Dataset snapshot was not updated. Cause of failure: " + e.getMessage(), null);
507+
"Dataset snapshot was not updated. Cause of failure: " + e.getMessage());
508508
}
509509
}
510510
}
@@ -791,7 +791,7 @@ public void run()
791791
DatasetDefinition dsDef = StudyManager.getInstance().getDatasetDefinitionByName(study, _def.getName());
792792
if (dsDef != null)
793793
new DatasetDefinition.DatasetAuditHandler(dsDef).addAuditEvent(context.getUser(), context.getContainer(), AuditBehaviorType.DETAILED,
794-
"Dataset snapshot was not updated. Cause of failure: " + errors.getMessage(), null);
794+
"Dataset snapshot was not updated. Cause of failure: " + errors.getMessage());
795795
}
796796
}
797797
}

‎study/src/org/labkey/study/model/DatasetDefinition.java‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,37 @@
3636
import org.labkey.api.collections.CaseInsensitiveHashMap;
3737
import org.labkey.api.collections.CaseInsensitiveHashSet;
3838
import org.labkey.api.collections.Sets;
39-
import org.labkey.api.data.*;
39+
import org.labkey.api.data.AuditConfigurable;
40+
import org.labkey.api.data.BaseColumnInfo;
41+
import org.labkey.api.data.BeanObjectFactory;
42+
import org.labkey.api.data.ColumnInfo;
43+
import org.labkey.api.data.ConnectionWrapper;
44+
import org.labkey.api.data.Container;
45+
import org.labkey.api.data.ContainerManager;
46+
import org.labkey.api.data.DataColumn;
47+
import org.labkey.api.data.DatabaseCache;
48+
import org.labkey.api.data.DatabaseTableType;
49+
import org.labkey.api.data.DbSchema;
50+
import org.labkey.api.data.DbScope;
4051
import org.labkey.api.data.DbScope.Transaction;
52+
import org.labkey.api.data.DisplayColumn;
53+
import org.labkey.api.data.DisplayColumnFactory;
54+
import org.labkey.api.data.ExceptionFramework;
55+
import org.labkey.api.data.JdbcType;
56+
import org.labkey.api.data.NullColumnInfo;
57+
import org.labkey.api.data.ObjectFactory;
58+
import org.labkey.api.data.PropertyManager;
59+
import org.labkey.api.data.RuntimeSQLException;
60+
import org.labkey.api.data.SQLFragment;
61+
import org.labkey.api.data.SchemaTableInfo;
62+
import org.labkey.api.data.SimpleFilter;
63+
import org.labkey.api.data.SqlExecutor;
64+
import org.labkey.api.data.SqlSelector;
65+
import org.labkey.api.data.Table;
66+
import org.labkey.api.data.TableInfo;
67+
import org.labkey.api.data.TableSelector;
68+
import org.labkey.api.data.Transient;
69+
import org.labkey.api.data.UpdateableTableInfo;
4170
import org.labkey.api.data.dialect.SqlDialect;
4271
import org.labkey.api.dataiterator.DataIterator;
4372
import org.labkey.api.dataiterator.DataIteratorBuilder;
@@ -1843,18 +1872,14 @@ else if (existingRecord != null && !existingRecord.isEmpty())
18431872
* @param requiredAuditType The expected audit behavior type. If this does not match the type set on the
18441873
* dataset, then the event will not be logged.
18451874
*/
1846-
public void addAuditEvent(User user, Container c, AuditBehaviorType requiredAuditType, String comment, @Nullable UploadLog ul)
1875+
public void addAuditEvent(User user, Container c, AuditBehaviorType requiredAuditType, String comment)
18471876

18481877
{
18491878
TableInfo table = _dataset.getTableInfo(user);
18501879
if (table != null && table.getEffectiveAuditBehavior((AuditBehaviorType)null) != requiredAuditType)
18511880
return;
18521881

18531882
DatasetAuditProvider.DatasetAuditEvent event = new DatasetAuditProvider.DatasetAuditEvent(c, comment, _dataset.getDatasetId());
1854-
if (ul != null)
1855-
{
1856-
event.setLsid(ul.getFilePath());
1857-
}
18581883
AuditLogService.get().addEvent(user, event);
18591884
}
18601885
}

‎study/src/org/labkey/study/model/StudyManager.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@
1616

1717
package org.labkey.study.model;
1818

19-
import org.apache.commons.collections4.MapUtils;
2019
import org.apache.commons.collections4.CollectionUtils;
2120
import org.apache.commons.collections4.ListUtils;
21+
import org.apache.commons.collections4.MapUtils;
2222
import org.apache.commons.lang3.StringUtils;
2323
import org.apache.commons.lang3.Strings;
2424
import org.apache.logging.log4j.Level;
@@ -226,12 +226,12 @@
226226
import java.util.stream.Collectors;
227227

228228
import static org.labkey.api.action.SpringActionController.ERROR_MSG;
229-
import static org.labkey.api.util.IntegerUtils.asInteger;
230229
import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PERSONNEL_TABLE_NAME;
231230
import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PRODUCT_ANTIGEN_TABLE_NAME;
232231
import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.PRODUCT_TABLE_NAME;
233232
import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.TREATMENT_PRODUCT_MAP_TABLE_NAME;
234233
import static org.labkey.api.studydesign.query.StudyDesignQuerySchema.TREATMENT_TABLE_NAME;
234+
import static org.labkey.api.util.IntegerUtils.asInteger;
235235

236236
public class StudyManager
237237
{
@@ -2566,7 +2566,7 @@ public void deleteDataset(StudyImpl study, User user, DatasetDefinition ds, bool
25662566

25672567
SchemaKey schemaPath = SchemaKey.fromParts(SCHEMA.getSchemaName());
25682568
QueryService.get().fireQueryDeleted(user, study.getContainer(), null, schemaPath, Collections.singleton(ds.getName()));
2569-
new DatasetDefinition.DatasetAuditHandler(ds).addAuditEvent(user, study.getContainer(), AuditBehaviorType.DETAILED, "Dataset deleted: " + ds.getName(), null);
2569+
new DatasetDefinition.DatasetAuditHandler(ds).addAuditEvent(user, study.getContainer(), AuditBehaviorType.DETAILED, "Dataset deleted: " + ds.getName());
25702570

25712571
transaction.addCommitTask(() ->
25722572
unindexDataset(ds),

0 commit comments

Comments
 (0)