From 076b04a895fc09090ece31e9a317e5f79c4127d5 Mon Sep 17 00:00:00 2001 From: David Milward Date: Tue, 26 Jun 2018 18:35:56 +0100 Subject: [PATCH 01/24] Adding in task name to collection and adapting gsp pages --- gradle.properties | 24 ++++++++++--------- grails-app/conf/application.yml | 2 +- .../RecordCollectionController.groovy | 3 ++- .../sentinel/RecordFileCommand.groovy | 1 + .../RecordCollectionGormEntity.groovy | 4 ++++ .../sentinel/BootStrap.groovy | 3 ++- .../sentinel/CsvImportProcessorService.groovy | 2 +- .../sentinel/CsvImportService.groovy | 4 +++- .../sentinel/ExcelImportService.groovy | 2 +- .../RecordCollectionGormService.groovy | 2 +- .../views/recordCollection/importCsv.gsp | 4 ++++ grails-app/views/recordCollection/index.gsp | 2 ++ .../sentinel/CsvImport.groovy | 2 +- 13 files changed, 36 insertions(+), 19 deletions(-) diff --git a/gradle.properties b/gradle.properties index aa3c774..5e6cf44 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,15 +1,17 @@ +#Tue Jun 26 14:44:22 BST 2018 +jsonViewsVersion=1.2.7 +webdriverBinariesGradleVersion=1.4 +grailsWrapperVersion=1.0.0 +poiVersion=3.17 grailsVersion=3.3.2 -gormVersion=6.1.8.RELEASE +hamcrestVersion=1.3 +seleniumVersion=3.6.0 gradleWrapperVersion=3.5 -droolsVersion=7.5.0.Final -okHttpVersion=3.9.1 -moshiVersion=1.5.0 -mysqlVersion=5.1.46 +gormVersion=6.1.8.RELEASE grailsExecutorVersion=0.4 -seleniumVersion=3.6.0 -assetPipelineVersion=2.14.8 -webdriverBinariesGradleVersion=1.4 -poiVersion=3.17 -jsonViewsVersion=1.2.7 +mysqlVersion=5.1.46 ersatzVersion=1.6.2 -hamcrestVersion=1.3 \ No newline at end of file +assetPipelineVersion=2.14.8 +okHttpVersion=3.9.1 +moshiVersion=1.5.0 +droolsVersion=7.5.0.Final diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index bc9f99d..0fc35c6 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -107,7 +107,7 @@ dataSource: environments: development: dataSource: - dbCreate: update + dbCreate: create url: '${JDBC_CONNECTION_STRING}' test: dataSource: diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index b81ec14..4b0867c 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -98,8 +98,9 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon log.debug 'Content Type {}', cmd.csvFile.contentType InputStream inputStream = cmd.csvFile.inputStream Integer batchSize = cmd.batchSize + String datasetName = cmd.datasetName CsvImport importService = csvImportByContentType (ImportContentType.of(cmd.csvFile.contentType)) - importService.save(inputStream, batchSize) + importService.save(inputStream, datasetName, batchSize) redirect controller: 'recordCollection', action: 'index' } diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordFileCommand.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordFileCommand.groovy index b54acc8..c8c7444 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordFileCommand.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordFileCommand.groovy @@ -8,6 +8,7 @@ import org.springframework.web.multipart.MultipartFile class RecordFileCommand implements Validateable { MultipartFile csvFile Integer batchSize = 100 + String datasetName static constraints = { batchSize nullable: false diff --git a/grails-app/domain/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntity.groovy b/grails-app/domain/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntity.groovy index 331cc04..90b964d 100644 --- a/grails-app/domain/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntity.groovy +++ b/grails-app/domain/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntity.groovy @@ -5,6 +5,8 @@ import grails.compiler.GrailsCompileStatic @GrailsCompileStatic class RecordCollectionGormEntity { + String datasetName + Date dateCreated Date lastUpdated @@ -17,6 +19,8 @@ class RecordCollectionGormEntity { static constraints = { records nullable: true mappings nullable: true + datasetName blank: true + datasetName nullable: true } static mapping = { diff --git a/grails-app/init/uk/co/metadataconsulting/sentinel/BootStrap.groovy b/grails-app/init/uk/co/metadataconsulting/sentinel/BootStrap.groovy index b454d23..cbb7bf5 100644 --- a/grails-app/init/uk/co/metadataconsulting/sentinel/BootStrap.groovy +++ b/grails-app/init/uk/co/metadataconsulting/sentinel/BootStrap.groovy @@ -17,7 +17,8 @@ class BootStrap { List mapping = mappingGormUrl() File f = new File('src/test/resources/DIDS_XMLExample_01.csv') InputStream inputStream = f.newInputStream() - csvImportService.save(inputStream, 100) + String datasetName = "SampleDataset" + csvImportService.save(inputStream, datasetName, 100) } List mappingGormUrl() { diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportProcessorService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportProcessorService.groovy index 7852bdd..a23f6dc 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportProcessorService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportProcessorService.groovy @@ -40,7 +40,7 @@ class CsvImportProcessorService implements GrailsConfigurationAware, CsvImportPr } @Override - int processInputStream(InputStream inputStream, Integer batchSize, Closure headerListClosure, Closure cls) { + int processInputStream(InputStream inputStream, Integer batchSize, Closure headerListClosure, Closure cls) { int processed = 0 BufferedReader br = new BufferedReader(new InputStreamReader(inputStream)) List lines = [] diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportService.groovy index ca5384e..1883bc6 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportService.groovy @@ -26,8 +26,10 @@ class CsvImportService implements CsvImport, Benchmark { @CompileDynamic @Override - void save(InputStream inputStream, Integer batchSize) { + void save(InputStream inputStream, String datasetName, Integer batchSize ) { + RecordCollectionGormEntity recordCollection = recordCollectionGormService.save() + recordCollection.datasetName = datasetName executorService.submit { log.info 'fetching validation rules' diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelImportService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelImportService.groovy index d859577..43d8878 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelImportService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelImportService.groovy @@ -20,7 +20,7 @@ class ExcelImportService implements CsvImport, Benchmark { @CompileDynamic @Override - void save(InputStream inputStream, Integer batchSize) { + void save(InputStream inputStream, String datasetName, Integer batchSize) { RecordCollectionGormEntity recordCollection = recordCollectionGormService.save() executorService.submit { diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionGormService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionGormService.groovy index 6cf2df9..fbd35a3 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionGormService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionGormService.groovy @@ -39,7 +39,7 @@ class RecordCollectionGormService implements GormErrorsMessage { @Transactional RecordCollectionGormEntity save() { RecordCollectionGormEntity recordCollection = new RecordCollectionGormEntity() - if ( !recordCollection.save() ) { + if ( !recordCollection.save(validate:false) ) { log.warn '{}', errorsMsg(recordCollection, messageSource) } recordCollection diff --git a/grails-app/views/recordCollection/importCsv.gsp b/grails-app/views/recordCollection/importCsv.gsp index 36c35c4..3491e13 100644 --- a/grails-app/views/recordCollection/importCsv.gsp +++ b/grails-app/views/recordCollection/importCsv.gsp @@ -23,6 +23,10 @@ +
+ +
diff --git a/grails-app/views/recordCollection/index.gsp b/grails-app/views/recordCollection/index.gsp index 8fe5b0b..0420dec 100644 --- a/grails-app/views/recordCollection/index.gsp +++ b/grails-app/views/recordCollection/index.gsp @@ -18,6 +18,7 @@ + @@ -25,6 +26,7 @@ +
${recordCollection.datasetName} ${recordCollection.lastUpdated} diff --git a/src/main/groovy/uk/co/metadataconsulting/sentinel/CsvImport.groovy b/src/main/groovy/uk/co/metadataconsulting/sentinel/CsvImport.groovy index 453c201..bb98c5b 100644 --- a/src/main/groovy/uk/co/metadataconsulting/sentinel/CsvImport.groovy +++ b/src/main/groovy/uk/co/metadataconsulting/sentinel/CsvImport.groovy @@ -4,5 +4,5 @@ import groovy.transform.CompileStatic @CompileStatic interface CsvImport { - void save(InputStream inputStream, Integer batchSize) + void save(InputStream inputStream, String datasetName, Integer batchSize) } \ No newline at end of file From 3f7ef93bceb233f4c438fe0dd71bcd7d865dacb7 Mon Sep 17 00:00:00 2001 From: David Milward Date: Thu, 28 Jun 2018 15:52:56 +0100 Subject: [PATCH 02/24] Added basic code to export validated record list as csv file --- .../RecordCollectionController.groovy | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index 4b0867c..be2569b 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -1,11 +1,14 @@ package uk.co.metadataconsulting.sentinel -import grails.config.Config -import grails.core.support.GrailsConfigurationAware + import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.springframework.context.MessageSource import uk.co.metadataconsulting.sentinel.modelcatalogue.ValidationRules +import grails.config.Config +import grails.core.support.GrailsConfigurationAware + +import static org.springframework.http.HttpStatus.OK @Slf4j @CompileStatic @@ -28,6 +31,8 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon RecordCollectionGormService recordCollectionGormService + RecordGormService recordGormService + RecordCollectionService recordCollectionService ExcelImportService excelImportService @@ -68,6 +73,36 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon ] } + def exportValidRecords(Long recordCollectionId){ + + String csvMimeType + String encoding + + final String filename = 'dataset_valid.csv' + List recordPortionMappingList = recordCollectionMappingGormService.findAllByRecordCollectionId(recordCollectionId) + + def validCsvExport = response.outputStream + response.status = OK.value() + response.contentType = "${csvMimeType};charset=${encoding}"; + response.setHeader "Content-disposition", "attachment; filename=${filename}" + + recordPortionMappingList.each { RecordPortionMapping record -> + + RecordGormEntity dataRecord = recordGormService.findById(record.id) + + if (dataRecord.validate()) { + validCsvExport << "${dataRecord}," + }else{ + println dataRecord + } + } + validCsvExport.flush() + validCsvExport.close() + + + redirect action: 'index', controller: 'record', params: [recordCollectionId: recordCollectionId] + } + def importCsv() { [:] } From 71e674fa1db5e0222d423bce2dfed3091d0df3e9 Mon Sep 17 00:00:00 2001 From: David Milward Date: Thu, 28 Jun 2018 18:00:01 +0100 Subject: [PATCH 03/24] Update to use the Excel plugin --- .../RecordCollectionController.groovy | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index be2569b..f2d699f 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -4,6 +4,8 @@ package uk.co.metadataconsulting.sentinel import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.springframework.context.MessageSource +import pl.touk.excel.export.WebXlsxExporter +import pl.touk.excel.export.XlsxExporter import uk.co.metadataconsulting.sentinel.modelcatalogue.ValidationRules import grails.config.Config import grails.core.support.GrailsConfigurationAware @@ -11,7 +13,6 @@ import grails.core.support.GrailsConfigurationAware import static org.springframework.http.HttpStatus.OK @Slf4j -@CompileStatic class RecordCollectionController implements ValidateableErrorsMessage, GrailsConfigurationAware { static allowedMethods = [ @@ -103,6 +104,36 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon redirect action: 'index', controller: 'record', params: [recordCollectionId: recordCollectionId] } + def exportValidExcel(Long recordCollectionId) { + + String csvMimeType + String encoding + + final String filename = 'dataset_valid.csv' + //List recordPortionMappingList = recordCollectionMappingGormService.findAllByRecordCollectionId(recordCollectionId) + List recordGormEntityList = recordGormService.findAllByRecordCollectionId(recordCollectionId) + List dataRecordsList = new ArrayList() + List headers = new ArrayList() + recordGormEntityList.each { RecordGormEntity record -> + record.portions.each { RecordPortionGormEntity portion -> + headers.add(portion.header) + } + } + + // new XlsxExporter('/tmp/myReportFile.xlsx'). + // add(dataRecordsList, properties).save() + new WebXlsxExporter().with { + setResponseHeaders(response) + fillHeader(headers) + add(dataRecordsList ) + save(response.outputStream) + } + + + redirect action: 'index', controller: 'record', params: [recordCollectionId: recordCollectionId] + } + + def importCsv() { [:] } From 98a94c767d4fb06308f0fc4b4abf654811f6d1f3 Mon Sep 17 00:00:00 2001 From: David Milward Date: Fri, 29 Jun 2018 11:36:42 +0100 Subject: [PATCH 04/24] Added in Excel Plugin --- .../sentinel/RecordIndexCommand.groovy | 1 + .../metadataconsulting/sentinel/UrlMappings.groovy | 1 + .../sentinel/RecordGormService.groovy | 7 +++++++ grails-app/views/record/index.gsp | 12 ++++++++---- grails-app/views/record/show.gsp | 1 + 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordIndexCommand.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordIndexCommand.groovy index ea73a52..7d98d71 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordIndexCommand.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordIndexCommand.groovy @@ -9,6 +9,7 @@ class RecordIndexCommand implements Validateable { Integer offset Integer max Long recordCollectionId + //String datasetName RecordCorrectnessDropdown correctness = RecordCorrectnessDropdown.ALL static constraints = { diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy index c5f665c..1a9d692 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy @@ -7,6 +7,7 @@ class UrlMappings { "/recordCollection/cloneMapping"(controller: 'recordCollection', action: 'cloneMapping') "/recordCollection/cloneSave"(controller: 'recordCollection', action: 'cloneSave', httpMethod: 'POST') "/recordCollection/validate"(controller: 'recordCollection', action: 'validate') + "/recordCollection/exportValidExcel"(controller: 'recordCollection', action: 'exportValidExcel') "/recordCollection/delete"(controller: 'recordCollection', action: 'delete') "/recordCollection/$recordCollectionId/mapping"(controller: 'recordCollection', action: 'headersMapping') "/recordCollectionMapping/catalogueElements/$dataModelId"(controller: 'recordCollectionMapping', action: 'catalogueElements') diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordGormService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordGormService.groovy index 1fc6833..a8aa83d 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordGormService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordGormService.groovy @@ -85,6 +85,13 @@ class RecordGormService implements GormErrorsMessage { query.list(paginationQuery.toMap()) } + @ReadOnly + List findAllByRecordCollectionId(Long recordCollectionId ) { + DetachedCriteria query = queryByRecordCollectionId(recordCollectionId) + return query as List + + } + @ReadOnly RecordGormEntity findById(Long recordId, List joinProperties = null) { DetachedCriteria query = queryById(recordId) diff --git a/grails-app/views/record/index.gsp b/grails-app/views/record/index.gsp index 2e0c5cf..97391db 100644 --- a/grails-app/views/record/index.gsp +++ b/grails-app/views/record/index.gsp @@ -10,10 +10,10 @@
  • - - - - + + + +
  • @@ -22,6 +22,9 @@ + + +
  • @@ -30,6 +33,7 @@
    • +
From fe540aac7091728ab43f7cdb4fd09694a8c67d6a Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 13:19:07 +0200 Subject: [PATCH 05/24] datasetName in a GormService and test --- .../sentinel/RecordFileCommand.groovy | 1 + .../RecordCollectionGormEntity.groovy | 3 +-- .../sentinel/CsvImportService.groovy | 3 +-- .../sentinel/ExcelImportService.groovy | 2 +- .../RecordCollectionGormService.groovy | 3 ++- ...CollectionControllerIntegrationSpec.groovy | 2 +- .../RecordControllerIntegrationSpec.groovy | 2 +- ...CollectionGormEntityConstraintsSpec.groovy | 22 +++++++++++++++++++ .../RecordFileCommandConstraintsSpec.groovy | 14 ++++++++++++ 9 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntityConstraintsSpec.groovy diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordFileCommand.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordFileCommand.groovy index c8c7444..14cee17 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordFileCommand.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordFileCommand.groovy @@ -11,6 +11,7 @@ class RecordFileCommand implements Validateable { String datasetName static constraints = { + datasetName nullable: false, blank: false batchSize nullable: false csvFile validator: { MultipartFile val, RecordFileCommand obj -> if ( val == null ) { diff --git a/grails-app/domain/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntity.groovy b/grails-app/domain/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntity.groovy index 90b964d..e7bff5d 100644 --- a/grails-app/domain/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntity.groovy +++ b/grails-app/domain/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntity.groovy @@ -19,8 +19,7 @@ class RecordCollectionGormEntity { static constraints = { records nullable: true mappings nullable: true - datasetName blank: true - datasetName nullable: true + datasetName nullable: false, blank: false } static mapping = { diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportService.groovy index 1883bc6..e8f3add 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/CsvImportService.groovy @@ -28,8 +28,7 @@ class CsvImportService implements CsvImport, Benchmark { @Override void save(InputStream inputStream, String datasetName, Integer batchSize ) { - RecordCollectionGormEntity recordCollection = recordCollectionGormService.save() - recordCollection.datasetName = datasetName + RecordCollectionGormEntity recordCollection = recordCollectionGormService.save(datasetName) executorService.submit { log.info 'fetching validation rules' diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelImportService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelImportService.groovy index 43d8878..58b60ea 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelImportService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelImportService.groovy @@ -21,7 +21,7 @@ class ExcelImportService implements CsvImport, Benchmark { @CompileDynamic @Override void save(InputStream inputStream, String datasetName, Integer batchSize) { - RecordCollectionGormEntity recordCollection = recordCollectionGormService.save() + RecordCollectionGormEntity recordCollection = recordCollectionGormService.save(datasetName) executorService.submit { log.info 'fetching validation rules' diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionGormService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionGormService.groovy index fbd35a3..7cbe09f 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionGormService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionGormService.groovy @@ -37,8 +37,9 @@ class RecordCollectionGormService implements GormErrorsMessage { } @Transactional - RecordCollectionGormEntity save() { + RecordCollectionGormEntity save(String datasetName) { RecordCollectionGormEntity recordCollection = new RecordCollectionGormEntity() + recordCollection.datasetName = datasetName if ( !recordCollection.save(validate:false) ) { log.warn '{}', errorsMsg(recordCollection, messageSource) } diff --git a/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/RecordCollectionControllerIntegrationSpec.groovy b/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/RecordCollectionControllerIntegrationSpec.groovy index a90a3a1..0deb0b6 100644 --- a/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/RecordCollectionControllerIntegrationSpec.groovy +++ b/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/RecordCollectionControllerIntegrationSpec.groovy @@ -50,7 +50,7 @@ class RecordCollectionControllerIntegrationSpec extends Specification { ruleFetcherService.metadataUrl = ersatz.httpUrl when: - RecordCollectionGormEntity recordCollection= recordCollectionGormService.save() + RecordCollectionGormEntity recordCollection= recordCollectionGormService.save("Test") then: recordCollectionGormService.count() == old(recordCollectionGormService.count()) + 1 diff --git a/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/RecordControllerIntegrationSpec.groovy b/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/RecordControllerIntegrationSpec.groovy index e23dc94..6b4ac73 100644 --- a/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/RecordControllerIntegrationSpec.groovy +++ b/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/RecordControllerIntegrationSpec.groovy @@ -57,7 +57,7 @@ class RecordControllerIntegrationSpec extends Specification { ruleFetcherService.metadataUrl = ersatz.httpUrl when: - RecordCollectionGormEntity recordCollection= recordCollectionGormService.save() + RecordCollectionGormEntity recordCollection= recordCollectionGormService.save("Test") then: recordCollectionGormService.count() == old(recordCollectionGormService.count()) + 1 diff --git a/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntityConstraintsSpec.groovy b/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntityConstraintsSpec.groovy new file mode 100644 index 0000000..c129fd4 --- /dev/null +++ b/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionGormEntityConstraintsSpec.groovy @@ -0,0 +1,22 @@ +package uk.co.metadataconsulting.sentinel + +import grails.testing.gorm.DomainUnitTest +import spock.lang.Specification + +class RecordCollectionGormEntityConstraintsSpec extends Specification implements DomainUnitTest { + + void 'verify datasetName is required and cannot be null'() { + when: + domain.datasetName = null + + then: + !domain.validate(['datasetName']) + + when: + domain.datasetName = '' + + then: + !domain.validate(['datasetName']) + } + +} diff --git a/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordFileCommandConstraintsSpec.groovy b/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordFileCommandConstraintsSpec.groovy index 69acc3b..a8a0195 100644 --- a/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordFileCommandConstraintsSpec.groovy +++ b/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordFileCommandConstraintsSpec.groovy @@ -18,4 +18,18 @@ class RecordFileCommandConstraintsSpec extends Specification { !cmd.validate(['batchSize']) cmd.errors['batchSize'].code == 'nullable' } + + void 'verify datasetName is required and cannot be null'() { + when: + cmd.datasetName = null + + then: + !cmd.validate(['datasetName']) + + when: + cmd.datasetName = '' + + then: + !cmd.validate(['datasetName']) + } } From fa81307bde6b21f59403bfa735312bd18057d09f Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 14:50:29 +0200 Subject: [PATCH 06/24] CSV Export --- .../RecordCollectionController.groovy | 70 ++++++++++--------- .../sentinel/UrlMappings.groovy | 1 + .../sentinel/ExcelGeneratorService.groovy | 0 .../RecordCollectionExportService.groovy | 30 ++++++++ .../sentinel/RecordGormService.groovy | 15 ++++ .../sentinel/RecordService.groovy | 57 ++++++++------- .../sentinel/RecordPortionUtils.groovy | 17 +++++ grails-app/views/record/index.gsp | 2 +- .../sentinel/RecordPortion.groovy | 10 +++ .../RecordCollectionExportRowView.groovy | 13 ++++ .../export/RecordCollectionExportView.groovy | 9 +++ ...lectionControllerAllowedMethodsSpec.groovy | 30 ++++++++ 12 files changed, 196 insertions(+), 58 deletions(-) create mode 100644 grails-app/services/uk/co/metadataconsulting/sentinel/ExcelGeneratorService.groovy create mode 100644 grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionExportService.groovy create mode 100644 grails-app/utils/uk/co/metadataconsulting/sentinel/RecordPortionUtils.groovy create mode 100644 src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy create mode 100644 src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportView.groovy diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index f2d699f..63d1ad6 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -1,11 +1,11 @@ package uk.co.metadataconsulting.sentinel - -import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.springframework.context.MessageSource -import pl.touk.excel.export.WebXlsxExporter -import pl.touk.excel.export.XlsxExporter +import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportRowView +import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportView + +//import pl.touk.excel.export.WebXlsxExporter import uk.co.metadataconsulting.sentinel.modelcatalogue.ValidationRules import grails.config.Config import grails.core.support.GrailsConfigurationAware @@ -24,6 +24,7 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon headersMapping: 'GET', cloneMapping: 'GET', cloneSave: 'POST', + exportValidRecords: 'GET' ] MessageSource messageSource @@ -42,15 +43,23 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon RuleFetcherService ruleFetcherService + + RecordCollectionExportService recordCollectionExportService + int defaultPaginationMax = 25 int defaultPaginationOffset = 0 + String csvMimeType + String encoding @Override void setConfiguration(Config co) { defaultPaginationMax = co.getProperty('sentinel.pagination.max', Integer, 25) defaultPaginationOffset = co.getProperty('sentinel.pagination.offset', Integer, 0) + csvMimeType = co.getProperty('grails.mime.types.csv', String, 'text/csv') + encoding = co.getProperty('grails.converters.encoding', String, 'UTF-8') } + def index() { Integer max = params.int('max') ?: defaultPaginationMax @@ -76,32 +85,30 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon def exportValidRecords(Long recordCollectionId){ - String csvMimeType - String encoding - final String filename = 'dataset_valid.csv' - List recordPortionMappingList = recordCollectionMappingGormService.findAllByRecordCollectionId(recordCollectionId) - - def validCsvExport = response.outputStream - response.status = OK.value() - response.contentType = "${csvMimeType};charset=${encoding}"; - response.setHeader "Content-disposition", "attachment; filename=${filename}" - recordPortionMappingList.each { RecordPortionMapping record -> - - RecordGormEntity dataRecord = recordGormService.findById(record.id) - - if (dataRecord.validate()) { - validCsvExport << "${dataRecord}," - }else{ - println dataRecord + def outs = response.outputStream + response.status = OK.value() + response.contentType = "${csvMimeType};charset=${encoding}" + response.setHeader "Content-disposition", "attachment; filename=${filename}" + final String separator = ';' + RecordCollectionExportView view = recordCollectionExportService.export(recordCollectionId) + List headers = view.headers + // TODO remove this line + headers = headers.collect { [it, it, it, it, it, it] }.flatten() + outs << "${headers.join(separator)}\n" + + if ( view.rows ) { + List portionsHeaders = view.rows.first().recordPortionList.collect { RecordPortion.toHeaderCsv(separator) } + String line = "${portionsHeaders.join(separator)}\n" + outs << line + view.rows.each { RecordCollectionExportRowView row -> + outs << "${row.toCsv(separator)}\n" } } - validCsvExport.flush() - validCsvExport.close() - - redirect action: 'index', controller: 'record', params: [recordCollectionId: recordCollectionId] + outs.flush() + outs.close() } def exportValidExcel(Long recordCollectionId) { @@ -122,18 +129,17 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon // new XlsxExporter('/tmp/myReportFile.xlsx'). // add(dataRecordsList, properties).save() - new WebXlsxExporter().with { - setResponseHeaders(response) - fillHeader(headers) - add(dataRecordsList ) - save(response.outputStream) - } +// new WebXlsxExporter().with { +// setResponseHeaders(response) +// fillHeader(headers) +// add(dataRecordsList ) +// save(response.outputStream) +// } redirect action: 'index', controller: 'record', params: [recordCollectionId: recordCollectionId] } - def importCsv() { [:] } diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy index 1a9d692..abf2ea5 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy @@ -7,6 +7,7 @@ class UrlMappings { "/recordCollection/cloneMapping"(controller: 'recordCollection', action: 'cloneMapping') "/recordCollection/cloneSave"(controller: 'recordCollection', action: 'cloneSave', httpMethod: 'POST') "/recordCollection/validate"(controller: 'recordCollection', action: 'validate') + "/recordCollection/exportValidRecords"(controller: 'recordCollection', action: 'exportValidRecords') "/recordCollection/exportValidExcel"(controller: 'recordCollection', action: 'exportValidExcel') "/recordCollection/delete"(controller: 'recordCollection', action: 'delete') "/recordCollection/$recordCollectionId/mapping"(controller: 'recordCollection', action: 'headersMapping') diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelGeneratorService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/ExcelGeneratorService.groovy new file mode 100644 index 0000000..e69de29 diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionExportService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionExportService.groovy new file mode 100644 index 0000000..a2c2631 --- /dev/null +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionExportService.groovy @@ -0,0 +1,30 @@ +package uk.co.metadataconsulting.sentinel + +import groovy.transform.CompileStatic +import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportRowView +import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportView + +@CompileStatic +class RecordCollectionExportService { + + RecordService recordService + + RecordGormService recordGormService + + RecordCollectionExportView export(Long recordCollectionId, RecordCorrectnessDropdown correctnes = RecordCorrectnessDropdown.VALID) { + List recordIds = recordService.findAllIdsByRecordCollectionId(recordCollectionId, correctnes, null) + List recordGormEntityList = recordGormService.findAllByIds(recordIds) + List rows = [] + List headers + if ( recordGormEntityList ) { + headers = recordGormEntityList.first().portions*.header + for ( RecordGormEntity recordGormEntity : recordGormEntityList ) { + List recordPortionList = recordGormEntity.portions.collect { RecordPortionGormEntity recordPortionGormEntity -> + RecordPortionUtils.of(recordPortionGormEntity) } + rows << new RecordCollectionExportRowView(recordPortionList: recordPortionList) + } + } + + new RecordCollectionExportView(rows: rows, headers: headers) + } +} \ No newline at end of file diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordGormService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordGormService.groovy index a8aa83d..d0497cd 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordGormService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordGormService.groovy @@ -111,4 +111,19 @@ class RecordGormService implements GormErrorsMessage { Number count() { RecordGormEntity.count() } + + @ReadOnly + List findAllByIds(List ids, List propertiesToJoin = ['portions']) { + DetachedCriteria query = queryAllByIds(ids) + if ( propertiesToJoin ) { + for ( String propertyName : propertiesToJoin ) { + query.join('portions') + } + } + query.list() + } + + DetachedCriteria queryAllByIds(List ids) { + RecordGormEntity.where { id in ids } + } } \ No newline at end of file diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy index 096b549..f927042 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy @@ -40,6 +40,8 @@ class RecordService { } as Set } + + @Transactional void validate(Long recordId, List recordPortionMappingList, Map validationRulesMap) { DetachedCriteria query = recordGormService.queryById(recordId) @@ -54,32 +56,37 @@ class RecordService { } List findAllByRecordCollectionId(Long recordCollectionId, RecordCorrectnessDropdown correctness, PaginationQuery paginationQuery) { - // TODO Do this with a query - DetachedCriteria query = recordGormService.queryByRecordCollectionId(recordCollectionId) - //query.join('portions') - - if ( correctness == RecordCorrectnessDropdown.ALL ) { - Map args = paginationQuery.toMap() - - List l = query.list(args) - - List ids = query.id().list(args) as List - Set invalidRecordIds = findAllInvalidRecordIds() - - return ids.collect { Long id -> - new RecordViewModel(id: id, valid: !invalidRecordIds.contains(id)) - } + switch (correctness) { + case RecordCorrectnessDropdown.ALL: + List ids = findAllIdsByRecordCollectionId(recordCollectionId, correctness, paginationQuery) + Set invalidRecordIds = findAllInvalidRecordIds() + return ids.collect { Long id -> + new RecordViewModel(id: id, valid: !invalidRecordIds.contains(id)) + } + case RecordCorrectnessDropdown.VALID: + return findAllIdsByRecordCollectionId(recordCollectionId, correctness, paginationQuery).collect { + new RecordViewModel(id: it, valid: true) + } + case RecordCorrectnessDropdown.INVALID: + return findAllIdsByRecordCollectionId(recordCollectionId, correctness, paginationQuery).collect { + new RecordViewModel(id: it, valid: false) + } } + } - boolean valid = validForCorrectnes(correctness) - - if ( valid ) { - return findAllValidRecords(recordCollectionId, paginationQuery).collect { - new RecordViewModel(id:it, valid: true) - } - } - return findAllInvalidRecords(recordCollectionId, paginationQuery).collect { - new RecordViewModel(id:it, valid: false) + List findAllIdsByRecordCollectionId(Long recordCollectionId, RecordCorrectnessDropdown correctness, PaginationQuery paginationQuery) { + switch (correctness) { + case RecordCorrectnessDropdown.ALL: + // TODO Do this with a query + DetachedCriteria query = recordGormService.queryByRecordCollectionId(recordCollectionId) + //query.join('portions') + Map args = paginationQuery?.toMap() + List ids = (args ? query.id().list(args) : query.id().list()) as List + return ids + case RecordCorrectnessDropdown.VALID: + return findAllValidRecords(recordCollectionId, paginationQuery) + case RecordCorrectnessDropdown.INVALID: + return findAllInvalidRecords(recordCollectionId, paginationQuery) } } @@ -98,7 +105,7 @@ class RecordService { return [] as List } DetachedCriteria query = queryValidRecords(recordCollectionId, validRecordIds) - query.id().list(paginationQuery.toMap()) as List + return (paginationQuery ? query.id().list(paginationQuery.toMap()) : query.id().list()) as List } List findAllInvalidRecords(Long recordCollectionId, PaginationQuery paginationQuery) { diff --git a/grails-app/utils/uk/co/metadataconsulting/sentinel/RecordPortionUtils.groovy b/grails-app/utils/uk/co/metadataconsulting/sentinel/RecordPortionUtils.groovy new file mode 100644 index 0000000..a10e249 --- /dev/null +++ b/grails-app/utils/uk/co/metadataconsulting/sentinel/RecordPortionUtils.groovy @@ -0,0 +1,17 @@ +package uk.co.metadataconsulting.sentinel + +import groovy.transform.CompileStatic + +@CompileStatic +class RecordPortionUtils { + + static RecordPortion of(RecordPortionGormEntity recordPortionGormEntity) { + new RecordPortion( + header: recordPortionGormEntity.header, + name: recordPortionGormEntity.name, + value: recordPortionGormEntity.value, + status: recordPortionGormEntity.status, + reason: recordPortionGormEntity.reason, + numberOfRulesValidatedAgainst: recordPortionGormEntity.numberOfRulesValidatedAgainst) + } +} diff --git a/grails-app/views/record/index.gsp b/grails-app/views/record/index.gsp index 97391db..578a28f 100644 --- a/grails-app/views/record/index.gsp +++ b/grails-app/views/record/index.gsp @@ -23,7 +23,7 @@ - + diff --git a/src/main/groovy/uk/co/metadataconsulting/sentinel/RecordPortion.groovy b/src/main/groovy/uk/co/metadataconsulting/sentinel/RecordPortion.groovy index 89e9415..ea42874 100644 --- a/src/main/groovy/uk/co/metadataconsulting/sentinel/RecordPortion.groovy +++ b/src/main/groovy/uk/co/metadataconsulting/sentinel/RecordPortion.groovy @@ -12,4 +12,14 @@ class RecordPortion { ValidationStatus status = ValidationStatus.NOT_VALIDATED String reason Integer numberOfRulesValidatedAgainst = 0 + + static String toHeaderCsv(String separator = ';') { + List l = ['name', 'value', 'status', 'reason', 'numberOfRulesValidatedAgainst'] + l.join(separator) + } + + String toCsv(String separator = ';') { + List l = [name ?: '', value ?: '', status.toString() ?: '', reason ?: '', numberOfRulesValidatedAgainst] + l.join(separator) + } } diff --git a/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy new file mode 100644 index 0000000..f6bdd8f --- /dev/null +++ b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy @@ -0,0 +1,13 @@ +package uk.co.metadataconsulting.sentinel.export + +import groovy.transform.CompileStatic +import uk.co.metadataconsulting.sentinel.RecordPortion + +@CompileStatic +class RecordCollectionExportRowView { + List recordPortionList + + String toCsv(String separator = ';') { + recordPortionList.collect { it.toCsv(separator) }.join(separator) + } +} diff --git a/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportView.groovy b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportView.groovy new file mode 100644 index 0000000..158a6cc --- /dev/null +++ b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportView.groovy @@ -0,0 +1,9 @@ +package uk.co.metadataconsulting.sentinel.export + +import groovy.transform.CompileStatic + +@CompileStatic +class RecordCollectionExportView { + List headers = [] + List rows = [] +} diff --git a/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionControllerAllowedMethodsSpec.groovy b/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionControllerAllowedMethodsSpec.groovy index 6f12553..48863d8 100644 --- a/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionControllerAllowedMethodsSpec.groovy +++ b/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionControllerAllowedMethodsSpec.groovy @@ -3,12 +3,42 @@ package uk.co.metadataconsulting.sentinel import grails.testing.web.controllers.ControllerUnitTest import spock.lang.Specification import spock.lang.Unroll +import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportView + import static javax.servlet.http.HttpServletResponse.SC_OK import static javax.servlet.http.HttpServletResponse.SC_METHOD_NOT_ALLOWED import static javax.servlet.http.HttpServletResponse.SC_MOVED_TEMPORARILY class RecordCollectionControllerAllowedMethodsSpec extends Specification implements ControllerUnitTest { + @Unroll + def "test RecordCollectionController.exportValidRecords does not accept #method requests"(String method) { + when: + request.method = method + controller.exportValidRecords() + + then: + response.status == SC_METHOD_NOT_ALLOWED + + where: + method << ['PATCH', 'DELETE', 'POST', 'PUT'] + } + + def "test RecordCollectionController.exportValidRecords accepts GET requests"() { + given: + controller.recordCollectionExportService = Stub(RecordCollectionExportService) { + export(_,_) >> new RecordCollectionExportView() + } + + when: + request.method = 'GET' + controller.exportValidRecords() + + then: + response.status == SC_OK + } + + @Unroll def "test RecordCollectionController.cloneSave does not accept #method requests"(String method) { when: From 3674587feba5f5b3c0d4004372502d092f5652cc Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:27:44 +0200 Subject: [PATCH 07/24] excel --- build.gradle | 4 + gradle.properties | 1 + grails-app/conf/application.yml | 11 +- .../RecordCollectionController.groovy | 120 +++++++++++++----- .../sentinel/UrlMappings.groovy | 2 +- .../ExportRecordCollectionCommand.groovy | 16 +++ grails-app/i18n/messages.properties | 7 + .../sentinel/export/ExportService.groovy | 41 ++++++ .../RecordCollectionExportService.groovy | 9 +- grails-app/views/record/index.gsp | 13 +- .../sentinel/RecordPortion.groovy | 12 +- .../sentinel/export/ExportFormat.groovy | 8 ++ .../RecordCollectionExportRowView.groovy | 4 + ...lectionControllerAllowedMethodsSpec.groovy | 9 +- .../sentinel/UrlMappingsSpec.groovy | 1 + ...ordCollectionCommandConstraintsSpec.groovy | 21 +++ 16 files changed, 224 insertions(+), 55 deletions(-) create mode 100644 grails-app/controllers/uk/co/metadataconsulting/sentinel/export/ExportRecordCollectionCommand.groovy create mode 100644 grails-app/services/uk/co/metadataconsulting/sentinel/export/ExportService.groovy rename grails-app/services/uk/co/metadataconsulting/sentinel/{ => export}/RecordCollectionExportService.groovy (74%) create mode 100644 src/main/groovy/uk/co/metadataconsulting/sentinel/export/ExportFormat.groovy create mode 100644 src/test/groovy/uk/co/metadataconsulting/sentinel/export/ExportRecordCollectionCommandConstraintsSpec.groovy diff --git a/build.gradle b/build.gradle index b75a725..f508c46 100644 --- a/build.gradle +++ b/build.gradle @@ -27,6 +27,7 @@ apply plugin: "org.grails.plugins.views-json" repositories { mavenLocal() maven { url "https://repo.grails.org/grails/core" } + jcenter() } dependencies { @@ -86,6 +87,9 @@ dependencies { testCompile "com.stehno.ersatz:ersatz:$ersatzVersion:safe@jar" testCompile "org.hamcrest:hamcrest-library:$hamcrestVersion" + + compile "builders.dsl:spreadsheet-builder-poi:$spreadsheetBuilderVersion" + compile "builders.dsl:spreadsheet-builder-groovy:$spreadsheetBuilderVersion" } bootRun { diff --git a/gradle.properties b/gradle.properties index 5e6cf44..54313da 100644 --- a/gradle.properties +++ b/gradle.properties @@ -15,3 +15,4 @@ assetPipelineVersion=2.14.8 okHttpVersion=3.9.1 moshiVersion=1.5.0 droolsVersion=7.5.0.Final +spreadsheetBuilderVersion=1.0.5 diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index 0fc35c6..d4fa511 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -65,6 +65,7 @@ grails: xml: - text/xml - application/xml + urlmapping: cache: maxsize: 1000 @@ -99,16 +100,16 @@ hibernate: dataSource: pooled: true jmxExport: true - driverClassName: '${JDBC_DRIVER}' - dialect: '${JDBC_DIALECT}' - username: '${JDBC_USERNAME}' - password: '${JDBC_PASSWORD}' + driverClassName: com.mysql.jdbc.Driver + dialect: org.hibernate.dialect.MySQL5InnoDBDialect + username: root + password: root environments: development: dataSource: dbCreate: create - url: '${JDBC_CONNECTION_STRING}' + url: jdbc:mysql://127.0.0.1:8889/metadatasentinel_dev test: dataSource: dbCreate: create-drop diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index 63d1ad6..95d6883 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -1,16 +1,18 @@ package uk.co.metadataconsulting.sentinel +import builders.dsl.spreadsheet.builder.poi.PoiSpreadsheetBuilder +import grails.config.Config +import grails.core.support.GrailsConfigurationAware import groovy.util.logging.Slf4j import org.springframework.context.MessageSource +import uk.co.metadataconsulting.sentinel.export.ExportRecordCollectionCommand +import uk.co.metadataconsulting.sentinel.export.ExportService import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportRowView +import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportService import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportView - -//import pl.touk.excel.export.WebXlsxExporter import uk.co.metadataconsulting.sentinel.modelcatalogue.ValidationRules -import grails.config.Config -import grails.core.support.GrailsConfigurationAware - import static org.springframework.http.HttpStatus.OK +import uk.co.metadataconsulting.sentinel.export.ExportFormat @Slf4j class RecordCollectionController implements ValidateableErrorsMessage, GrailsConfigurationAware { @@ -24,7 +26,7 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon headersMapping: 'GET', cloneMapping: 'GET', cloneSave: 'POST', - exportValidRecords: 'GET' + export: 'GET' ] MessageSource messageSource @@ -43,23 +45,21 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon RuleFetcherService ruleFetcherService - RecordCollectionExportService recordCollectionExportService + ExportService exportService + int defaultPaginationMax = 25 int defaultPaginationOffset = 0 - String csvMimeType - String encoding + String separator @Override void setConfiguration(Config co) { + separator = co.getProperty('export.csv.separator', String, ';') defaultPaginationMax = co.getProperty('sentinel.pagination.max', Integer, 25) defaultPaginationOffset = co.getProperty('sentinel.pagination.offset', Integer, 0) - csvMimeType = co.getProperty('grails.mime.types.csv', String, 'text/csv') - encoding = co.getProperty('grails.converters.encoding', String, 'UTF-8') } - def index() { Integer max = params.int('max') ?: defaultPaginationMax @@ -83,30 +83,80 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon ] } - def exportValidRecords(Long recordCollectionId){ + def export(ExportRecordCollectionCommand cmd) { + + if ( cmd.hasErrors() ) { + // TODO Maybe display an error message with flash.error and redirect to HomePage + return + } - final String filename = 'dataset_valid.csv' + RecordCollectionGormEntity collectionGormEntity = recordCollectionGormService.find(cmd.recordCollectionId) + if ( !collectionGormEntity ) { + // TODO Maybe display an error message with flash.error and redirect to HomePage + return + } + final String filenameprefix = collectionGormEntity.datasetName + response.status = OK.value() + RecordCollectionExportView view = recordCollectionExportService.export(cmd.recordCollectionId) def outs = response.outputStream - response.status = OK.value() - response.contentType = "${csvMimeType};charset=${encoding}" - response.setHeader "Content-disposition", "attachment; filename=${filename}" - final String separator = ';' - RecordCollectionExportView view = recordCollectionExportService.export(recordCollectionId) List headers = view.headers - // TODO remove this line - headers = headers.collect { [it, it, it, it, it, it] }.flatten() - outs << "${headers.join(separator)}\n" - - if ( view.rows ) { - List portionsHeaders = view.rows.first().recordPortionList.collect { RecordPortion.toHeaderCsv(separator) } - String line = "${portionsHeaders.join(separator)}\n" - outs << line - view.rows.each { RecordCollectionExportRowView row -> - outs << "${row.toCsv(separator)}\n" - } + List portionsHeaders = view.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + String filename = "${filenameprefix}.${exportService.fileExtensionForFormat(cmd.format)}" + response.setHeader "Content-disposition", "attachment; filename=${filename}" + response.contentType = exportService.mimeTypeForFormat(cmd.format) + switch (cmd.format) { + case ExportFormat.CSV: + // TODO remove this line? + headers = headers.collect { [it, it, it, it, it, it] }.flatten() + outs << "${headers.join(separator)}\n" + if ( view.rows ) { + String line = "${portionsHeaders.join(separator)}\n" + outs << line + view.rows.each { RecordCollectionExportRowView row -> + outs << "${row.toCsv(separator)}\n" + } + } + break + case ExportFormat.XLSX: + + PoiSpreadsheetBuilder.create(outs).build { + sheet(collectionGormEntity.datasetName) { + row { + for ( String header : headers ) { + cell { + value header + colspan RecordPortion.toHeaderList().size() + } + } + } + if ( portionsHeaders ) { + row { + for ( String recordPortionHeader : portionsHeaders ) { + cell { + value messageSource.getMessage("export.header.${recordPortionHeader}".toString(), + [] as Object[], + recordPortionHeader, + request.locale) + } + } + } + } + if ( view.rows ) { + for (RecordCollectionExportRowView rowView : view.rows ) { + row { + for (String val : rowView.toList()) { + cell { + value val + } + } + } + } + } + } + } + break } - outs.flush() outs.close() } @@ -127,8 +177,8 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } } - // new XlsxExporter('/tmp/myReportFile.xlsx'). - // add(dataRecordsList, properties).save() + // new XlsxExporter('/tmp/myReportFile.xlsx'). + // add(dataRecordsList, properties).save() // new WebXlsxExporter().with { // setResponseHeaders(response) // fillHeader(headers) @@ -194,9 +244,9 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon List dataModelList = ruleFetcherService.fetchDataModels()?.dataModels if ( !dataModelList ) { flash.error = messageSource.getMessage('dataModel.couldNotLoad', [] as Object[], 'Could not load data Models', request.locale) - } + } [ - dataModelList: dataModelList, + dataModelList: dataModelList, recordCollectionId: recordCollectionId, recordPortionMappingList: recordCollectionMappingGormService.findAllByRecordCollectionId(recordCollectionId) ] diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy index abf2ea5..f042399 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy @@ -7,7 +7,7 @@ class UrlMappings { "/recordCollection/cloneMapping"(controller: 'recordCollection', action: 'cloneMapping') "/recordCollection/cloneSave"(controller: 'recordCollection', action: 'cloneSave', httpMethod: 'POST') "/recordCollection/validate"(controller: 'recordCollection', action: 'validate') - "/recordCollection/exportValidRecords"(controller: 'recordCollection', action: 'exportValidRecords') + "/recordCollection/export"(controller: 'recordCollection', action: 'export') "/recordCollection/exportValidExcel"(controller: 'recordCollection', action: 'exportValidExcel') "/recordCollection/delete"(controller: 'recordCollection', action: 'delete') "/recordCollection/$recordCollectionId/mapping"(controller: 'recordCollection', action: 'headersMapping') diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/export/ExportRecordCollectionCommand.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/export/ExportRecordCollectionCommand.groovy new file mode 100644 index 0000000..7276ac7 --- /dev/null +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/export/ExportRecordCollectionCommand.groovy @@ -0,0 +1,16 @@ +package uk.co.metadataconsulting.sentinel.export + +import grails.compiler.GrailsCompileStatic +import grails.validation.Validateable +import uk.co.metadataconsulting.sentinel.export.ExportFormat + +@GrailsCompileStatic +class ExportRecordCollectionCommand implements Validateable { + + Long recordCollectionId + ExportFormat format = ExportFormat.XLSX + + static constraints = { + recordCollectionId nullable: false + } +} \ No newline at end of file diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties index b045136..335748c 100644 --- a/grails-app/i18n/messages.properties +++ b/grails-app/i18n/messages.properties @@ -54,3 +54,10 @@ typeMismatch.java.lang.Short=Property {0} must be a valid number typeMismatch.java.math.BigDecimal=Property {0} must be a valid number typeMismatch.java.math.BigInteger=Property {0} must be a valid number typeMismatch=Property {0} is type-mismatched + + +export.header.name=Name +export.header.value=Value +export.header.status=Status +export.header.reason=Failure Reason +export.header.numberOfRulesValidatedAgainst=# Rules Validated Against diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/export/ExportService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/export/ExportService.groovy new file mode 100644 index 0000000..9b4dcb2 --- /dev/null +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/export/ExportService.groovy @@ -0,0 +1,41 @@ +package uk.co.metadataconsulting.sentinel.export + +import grails.config.Config +import grails.core.support.GrailsConfigurationAware +import groovy.transform.CompileStatic +import uk.co.metadataconsulting.sentinel.export.ExportFormat + +@CompileStatic +class ExportService implements GrailsConfigurationAware { + String xlsxMimeType + String xlsMimeType + String csvMimeType + String encoding + + @Override + void setConfiguration(Config co) { + csvMimeType = co.getProperty('grails.mime.types.csv', String, 'text/csv') + xlsMimeType = co.getProperty('grails.mime.types.xlsMimeType', String,'application/vnd.ms-excel') + xlsxMimeType = co.getProperty('grails.mime.types.xlsxMimeType', String, 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') + encoding = co.getProperty('grails.converters.encoding', String, 'UTF-8') + } + + String fileExtensionForFormat(ExportFormat format) { + switch (format) { + case ExportFormat.CSV: + return 'csv' + + case ExportFormat.XLSX: + return "xlsx" + } + } + String mimeTypeForFormat(ExportFormat format) { + switch (format) { + case ExportFormat.CSV: + return "${csvMimeType};charset=${encoding}" + + case ExportFormat.XLSX: + return "${xlsxMimeType};charset=${encoding}" + } + } +} \ No newline at end of file diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionExportService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportService.groovy similarity index 74% rename from grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionExportService.groovy rename to grails-app/services/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportService.groovy index a2c2631..451436b 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordCollectionExportService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportService.groovy @@ -1,6 +1,13 @@ -package uk.co.metadataconsulting.sentinel +package uk.co.metadataconsulting.sentinel.export import groovy.transform.CompileStatic +import uk.co.metadataconsulting.sentinel.RecordCorrectnessDropdown +import uk.co.metadataconsulting.sentinel.RecordGormEntity +import uk.co.metadataconsulting.sentinel.RecordGormService +import uk.co.metadataconsulting.sentinel.RecordPortion +import uk.co.metadataconsulting.sentinel.RecordPortionGormEntity +import uk.co.metadataconsulting.sentinel.RecordPortionUtils +import uk.co.metadataconsulting.sentinel.RecordService import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportRowView import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportView diff --git a/grails-app/views/record/index.gsp b/grails-app/views/record/index.gsp index 578a28f..b0440aa 100644 --- a/grails-app/views/record/index.gsp +++ b/grails-app/views/record/index.gsp @@ -1,4 +1,6 @@ <%@ page import="uk.co.metadataconsulting.sentinel.RecordCorrectnessDropdown" %> +<%@ page import="uk.co.metadataconsulting.sentinel.export.ExportFormat" %> + Records @@ -23,10 +25,13 @@ - - - - +
  • + + + + + +
  • diff --git a/src/main/groovy/uk/co/metadataconsulting/sentinel/RecordPortion.groovy b/src/main/groovy/uk/co/metadataconsulting/sentinel/RecordPortion.groovy index ea42874..7f2b36d 100644 --- a/src/main/groovy/uk/co/metadataconsulting/sentinel/RecordPortion.groovy +++ b/src/main/groovy/uk/co/metadataconsulting/sentinel/RecordPortion.groovy @@ -13,13 +13,15 @@ class RecordPortion { String reason Integer numberOfRulesValidatedAgainst = 0 - static String toHeaderCsv(String separator = ';') { - List l = ['name', 'value', 'status', 'reason', 'numberOfRulesValidatedAgainst'] - l.join(separator) + static List toHeaderList() { + ['name', 'value', 'status', 'reason', 'numberOfRulesValidatedAgainst'] + } + + List toList() { + [name ?: '', value ?: '', status.toString() ?: '', reason ?: '', "${numberOfRulesValidatedAgainst}".toString()] } String toCsv(String separator = ';') { - List l = [name ?: '', value ?: '', status.toString() ?: '', reason ?: '', numberOfRulesValidatedAgainst] - l.join(separator) + toList().join(separator) } } diff --git a/src/main/groovy/uk/co/metadataconsulting/sentinel/export/ExportFormat.groovy b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/ExportFormat.groovy new file mode 100644 index 0000000..e6b3d02 --- /dev/null +++ b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/ExportFormat.groovy @@ -0,0 +1,8 @@ +package uk.co.metadataconsulting.sentinel.export + +import groovy.transform.CompileStatic + +@CompileStatic +enum ExportFormat { + CSV, XLSX +} diff --git a/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy index f6bdd8f..2ac24b6 100644 --- a/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy +++ b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy @@ -10,4 +10,8 @@ class RecordCollectionExportRowView { String toCsv(String separator = ';') { recordPortionList.collect { it.toCsv(separator) }.join(separator) } + + List toList() { + recordPortionList.collect { it.toList() }.flatten() as List + } } diff --git a/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionControllerAllowedMethodsSpec.groovy b/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionControllerAllowedMethodsSpec.groovy index 48863d8..924177e 100644 --- a/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionControllerAllowedMethodsSpec.groovy +++ b/src/test/groovy/uk/co/metadataconsulting/sentinel/RecordCollectionControllerAllowedMethodsSpec.groovy @@ -3,6 +3,7 @@ package uk.co.metadataconsulting.sentinel import grails.testing.web.controllers.ControllerUnitTest import spock.lang.Specification import spock.lang.Unroll +import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportService import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportView import static javax.servlet.http.HttpServletResponse.SC_OK @@ -12,10 +13,10 @@ import static javax.servlet.http.HttpServletResponse.SC_MOVED_TEMPORARILY class RecordCollectionControllerAllowedMethodsSpec extends Specification implements ControllerUnitTest { @Unroll - def "test RecordCollectionController.exportValidRecords does not accept #method requests"(String method) { + def "test RecordCollectionController.export does not accept #method requests"(String method) { when: request.method = method - controller.exportValidRecords() + controller.export() then: response.status == SC_METHOD_NOT_ALLOWED @@ -24,7 +25,7 @@ class RecordCollectionControllerAllowedMethodsSpec extends Specification impleme method << ['PATCH', 'DELETE', 'POST', 'PUT'] } - def "test RecordCollectionController.exportValidRecords accepts GET requests"() { + def "test RecordCollectionController.export accepts GET requests"() { given: controller.recordCollectionExportService = Stub(RecordCollectionExportService) { export(_,_) >> new RecordCollectionExportView() @@ -32,7 +33,7 @@ class RecordCollectionControllerAllowedMethodsSpec extends Specification impleme when: request.method = 'GET' - controller.exportValidRecords() + controller.export() then: response.status == SC_OK diff --git a/src/test/groovy/uk/co/metadataconsulting/sentinel/UrlMappingsSpec.groovy b/src/test/groovy/uk/co/metadataconsulting/sentinel/UrlMappingsSpec.groovy index edab982..c7d9f65 100644 --- a/src/test/groovy/uk/co/metadataconsulting/sentinel/UrlMappingsSpec.groovy +++ b/src/test/groovy/uk/co/metadataconsulting/sentinel/UrlMappingsSpec.groovy @@ -12,6 +12,7 @@ class UrlMappingsSpec extends Specification implements UrlMappingsUnitTest Date: Fri, 29 Jun 2018 17:28:25 +0200 Subject: [PATCH 08/24] remove export valid excel method --- .../RecordCollectionController.groovy | 29 ------------------- .../sentinel/UrlMappings.groovy | 1 - 2 files changed, 30 deletions(-) diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index 95d6883..5aeee6e 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -161,35 +161,6 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon outs.close() } - def exportValidExcel(Long recordCollectionId) { - - String csvMimeType - String encoding - - final String filename = 'dataset_valid.csv' - //List recordPortionMappingList = recordCollectionMappingGormService.findAllByRecordCollectionId(recordCollectionId) - List recordGormEntityList = recordGormService.findAllByRecordCollectionId(recordCollectionId) - List dataRecordsList = new ArrayList() - List headers = new ArrayList() - recordGormEntityList.each { RecordGormEntity record -> - record.portions.each { RecordPortionGormEntity portion -> - headers.add(portion.header) - } - } - - // new XlsxExporter('/tmp/myReportFile.xlsx'). - // add(dataRecordsList, properties).save() -// new WebXlsxExporter().with { -// setResponseHeaders(response) -// fillHeader(headers) -// add(dataRecordsList ) -// save(response.outputStream) -// } - - - redirect action: 'index', controller: 'record', params: [recordCollectionId: recordCollectionId] - } - def importCsv() { [:] } diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy index f042399..3fc3e06 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/UrlMappings.groovy @@ -8,7 +8,6 @@ class UrlMappings { "/recordCollection/cloneSave"(controller: 'recordCollection', action: 'cloneSave', httpMethod: 'POST') "/recordCollection/validate"(controller: 'recordCollection', action: 'validate') "/recordCollection/export"(controller: 'recordCollection', action: 'export') - "/recordCollection/exportValidExcel"(controller: 'recordCollection', action: 'exportValidExcel') "/recordCollection/delete"(controller: 'recordCollection', action: 'delete') "/recordCollection/$recordCollectionId/mapping"(controller: 'recordCollection', action: 'headersMapping') "/recordCollectionMapping/catalogueElements/$dataModelId"(controller: 'recordCollectionMapping', action: 'catalogueElements') From a7b01625d04e5a8fa8a1891e498ba80ebe921fc2 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:32:03 +0200 Subject: [PATCH 09/24] fix unit test --- .../uk/co/metadataconsulting/sentinel/XlxReaderSpec.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/groovy/uk/co/metadataconsulting/sentinel/XlxReaderSpec.groovy b/src/test/groovy/uk/co/metadataconsulting/sentinel/XlxReaderSpec.groovy index 374ade8..0193821 100644 --- a/src/test/groovy/uk/co/metadataconsulting/sentinel/XlxReaderSpec.groovy +++ b/src/test/groovy/uk/co/metadataconsulting/sentinel/XlxReaderSpec.groovy @@ -13,15 +13,15 @@ class XlxReaderSpec extends Specification { f.exists() when: - List> lines = [] + List> lines = [] Closure cls = { List values -> lines << values } ExcelReader.read(f.newInputStream(), 0, true, null, cls) then: noExceptionThrown() lines.size() == 2 - lines.each { List line -> - assert line.any { String str -> str} + lines.each { List line -> + assert line.any { it } } } From f3f2ec3cc267dc23ae2a8aefa42f511d47dc0504 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:35:24 +0200 Subject: [PATCH 10/24] revert JDBC_ accidental change --- grails-app/conf/application.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index d4fa511..3b9d06c 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -100,16 +100,16 @@ hibernate: dataSource: pooled: true jmxExport: true - driverClassName: com.mysql.jdbc.Driver - dialect: org.hibernate.dialect.MySQL5InnoDBDialect - username: root - password: root + driverClassName: '${JDBC_DRIVER}' + dialect: '${JDBC_DIALECT}' + username: '${JDBC_USERNAME}' + password: '${JDBC_PASSWORD}' environments: development: dataSource: dbCreate: create - url: jdbc:mysql://127.0.0.1:8889/metadatasentinel_dev + url: '${JDBC_CONNECTION_STRING}' test: dataSource: dbCreate: create-drop From b1782b8a03fd38edb31985fdfece7b7b9954a442 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:40:09 +0200 Subject: [PATCH 11/24] Define chromedriver version in gradle.properties --- build.gradle | 2 +- gradle.properties | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f508c46..e53b77f 100644 --- a/build.gradle +++ b/build.gradle @@ -99,8 +99,8 @@ bootRun { } webdriverBinaries { - chromedriver '2.36' geckodriver '0.18.0' + chromedriver "${chromedriverVersion}" } tasks.withType(Test) { systemProperties System.properties diff --git a/gradle.properties b/gradle.properties index 54313da..577bcde 100644 --- a/gradle.properties +++ b/gradle.properties @@ -15,4 +15,5 @@ assetPipelineVersion=2.14.8 okHttpVersion=3.9.1 moshiVersion=1.5.0 droolsVersion=7.5.0.Final +chromedriverVersion=2.40 spreadsheetBuilderVersion=1.0.5 From 6a4ada9dbe6aa7a1e866e6340067bb0dc7ea6ddb Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:40:25 +0200 Subject: [PATCH 12/24] Define gecko driver version in gradle.properties --- build.gradle | 2 +- gradle.properties | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e53b77f..a5c68dc 100644 --- a/build.gradle +++ b/build.gradle @@ -99,8 +99,8 @@ bootRun { } webdriverBinaries { - geckodriver '0.18.0' chromedriver "${chromedriverVersion}" + geckodriver "${geckodriverVersion}" } tasks.withType(Test) { systemProperties System.properties diff --git a/gradle.properties b/gradle.properties index 577bcde..10e8d66 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,4 +16,5 @@ okHttpVersion=3.9.1 moshiVersion=1.5.0 droolsVersion=7.5.0.Final chromedriverVersion=2.40 +geckodriverVersion=0.21.0 spreadsheetBuilderVersion=1.0.5 From 5e1ca3b2037b04fb827807e4046112781fe047f0 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:40:49 +0200 Subject: [PATCH 13/24] remove spring.transactionaManagement.proxies conf --- grails-app/conf/application.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index 3b9d06c..6084d5e 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -3,9 +3,6 @@ grails: profile: web codegen: defaultPackage: uk.co.metadataconsulting.sentinel - spring: - transactionManagement: - proxies: false gorm: reactor: # Whether to translate GORM events into Reactor events From 14cb7a607ca9b16a6170c3bd6dd5a0c0d1797538 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:41:00 +0200 Subject: [PATCH 14/24] Define firefox headless geb.env --- src/integration-test/resources/GebConfig.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/integration-test/resources/GebConfig.groovy b/src/integration-test/resources/GebConfig.groovy index 0d97e35..95c7681 100644 --- a/src/integration-test/resources/GebConfig.groovy +++ b/src/integration-test/resources/GebConfig.groovy @@ -1,6 +1,7 @@ import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.chrome.ChromeOptions import org.openqa.selenium.firefox.FirefoxDriver +import org.openqa.selenium.firefox.FirefoxOptions environments { @@ -17,6 +18,15 @@ environments { new ChromeDriver(o) } } + + // run via “./gradlew -Dgeb.env=firefoxHeadless iT” + firefoxHeadless { + driver = { + FirefoxOptions o = new FirefoxOptions() + o.addArguments('-headless') + new FirefoxDriver(o) + } + } // run via “./gradlew -Dgeb.env=firefox iT” firefox { From 65d8af526f9de1fe4414ee0fbc9262c5d3b32cc7 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:41:28 +0200 Subject: [PATCH 15/24] Bump up Grails to 3.3.6 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 10e8d66..b83edd7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,7 +3,7 @@ jsonViewsVersion=1.2.7 webdriverBinariesGradleVersion=1.4 grailsWrapperVersion=1.0.0 poiVersion=3.17 -grailsVersion=3.3.2 +grailsVersion=3.3.6 hamcrestVersion=1.3 seleniumVersion=3.6.0 gradleWrapperVersion=3.5 From 1e0736e80fafbf4704813d933baccb992d9b08d4 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:41:42 +0200 Subject: [PATCH 16/24] Bump up GORM to 6.1.10 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b83edd7..7987827 100644 --- a/gradle.properties +++ b/gradle.properties @@ -7,7 +7,7 @@ grailsVersion=3.3.6 hamcrestVersion=1.3 seleniumVersion=3.6.0 gradleWrapperVersion=3.5 -gormVersion=6.1.8.RELEASE +gormVersion=6.1.10.RELEASE grailsExecutorVersion=0.4 mysqlVersion=5.1.46 ersatzVersion=1.6.2 From bac226479f05cb5d0eee760bc1093af6cf690f67 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:47:44 +0200 Subject: [PATCH 17/24] Fix integration test It need update to use the new method signature --- .../CsvImportServiceIntegrationSpec.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/CsvImportServiceIntegrationSpec.groovy b/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/CsvImportServiceIntegrationSpec.groovy index 62867e2..62e90d5 100644 --- a/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/CsvImportServiceIntegrationSpec.groovy +++ b/src/integration-test/groovy/uk.co.metadataconsulting.sentinel/CsvImportServiceIntegrationSpec.groovy @@ -32,7 +32,7 @@ class CsvImportServiceIntegrationSpec extends Specification { expectedNumberOfRows when: - csvImportService.save(f.newInputStream(), 50) + csvImportService.save(f.newInputStream(), "DIDS_XMLExample_20", 50) then: recordCollectionGormService.count() == old(recordCollectionGormService.count()) + 1 From 749d9b383c9d6e5a9aa59846fca9333b20a9ba14 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 17:47:59 +0200 Subject: [PATCH 18/24] Force tests to use grails.env=test --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index a5c68dc..13df05f 100644 --- a/build.gradle +++ b/build.gradle @@ -104,6 +104,7 @@ webdriverBinaries { } tasks.withType(Test) { systemProperties System.properties + systemProperty "grails.env", 'test' systemProperty "geb.env", System.getProperty('geb.env') systemProperty "geb.build.reportsDir", reporting.file("geb/integrationTest") beforeTest { descriptor -> logger.quiet " -- $descriptor" } From 2ca566d42ad558c4f8d7108c852efa515ce4b01b Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 18:13:38 +0200 Subject: [PATCH 19/24] Three tabs --- .../RecordCollectionController.groovy | 92 ++++++++++++++++++- .../sentinel/export/ExportFormat.groovy | 2 +- 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index 5aeee6e..214496c 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -98,7 +98,8 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon final String filenameprefix = collectionGormEntity.datasetName response.status = OK.value() - RecordCollectionExportView view = recordCollectionExportService.export(cmd.recordCollectionId) + RecordCorrectnessDropdown recordCorrectnessDropdown = RecordCorrectnessDropdown.ALL + RecordCollectionExportView view = recordCollectionExportService.export(cmd.recordCollectionId, recordCorrectnessDropdown) def outs = response.outputStream List headers = view.headers List portionsHeaders = view.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() @@ -119,9 +120,40 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } break case ExportFormat.XLSX: - PoiSpreadsheetBuilder.create(outs).build { - sheet(collectionGormEntity.datasetName) { + sheet(messageSource.getMessage("export.valid".toString(), [] as Object[], 'Valid', request.locale)) { + row { + for ( String header : headers ) { + cell { + value header + colspan RecordPortion.toHeaderList().size() + } + } + } + if ( portionsHeaders ) { + row { + for ( String recordPortionHeader : portionsHeaders ) { + cell { + value messageSource.getMessage("export.header.${recordPortionHeader}".toString(), + [] as Object[], + recordPortionHeader, + request.locale) + } + } + } + } + if ( view.rows ) { + for (RecordCollectionExportRowView rowView : findAllValid(view.rows) ) { + row { + for (String val : rowView.toList()) { + cell { + value val + } + } + } } + } + } + sheet(messageSource.getMessage("export.invalid".toString(), [] as Object[], 'Invalid', request.locale)) { row { for ( String header : headers ) { cell { @@ -143,7 +175,40 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } } if ( view.rows ) { - for (RecordCollectionExportRowView rowView : view.rows ) { + for (RecordCollectionExportRowView rowView : findAllInvalid(view.rows) ) { + row { + for (String val : rowView.toList()) { + cell { + value val + } + } + } + } + } + } + sheet(messageSource.getMessage("export.notValidated".toString(), [] as Object[], 'Not Validated', request.locale)) { + row { + for ( String header : headers ) { + cell { + value header + colspan RecordPortion.toHeaderList().size() + } + } + } + if ( portionsHeaders ) { + row { + for ( String recordPortionHeader : portionsHeaders ) { + cell { + value messageSource.getMessage("export.header.${recordPortionHeader}".toString(), + [] as Object[], + recordPortionHeader, + request.locale) + } + } + } + } + if ( view.rows ) { + for (RecordCollectionExportRowView rowView : findAllNotValidated(view.rows) ) { row { for (String val : rowView.toList()) { cell { @@ -155,12 +220,31 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } } } + break } outs.flush() outs.close() } + private List findAllValid(List all) { + all.findAll { RecordCollectionExportRowView row -> + !row.recordPortionList.any { it.status == ValidationStatus.INVALID } && row.recordPortionList.any { it.status == ValidationStatus.VALID } + } + } + + private List findAllInvalid(List all) { + all.findAll { RecordCollectionExportRowView row -> + row.recordPortionList.any { it.status == ValidationStatus.INVALID } + } + } + + private List findAllNotValidated(List all) { + all.findAll { RecordCollectionExportRowView row -> + row.recordPortionList.every { it.status == ValidationStatus.NOT_VALIDATED } + } + } + def importCsv() { [:] } diff --git a/src/main/groovy/uk/co/metadataconsulting/sentinel/export/ExportFormat.groovy b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/ExportFormat.groovy index e6b3d02..0dde274 100644 --- a/src/main/groovy/uk/co/metadataconsulting/sentinel/export/ExportFormat.groovy +++ b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/ExportFormat.groovy @@ -4,5 +4,5 @@ import groovy.transform.CompileStatic @CompileStatic enum ExportFormat { - CSV, XLSX + XLSX, CSV } From 197c123d565b70bfc896950cdf94d55e281066b0 Mon Sep 17 00:00:00 2001 From: sdelamo Date: Fri, 29 Jun 2018 18:23:44 +0200 Subject: [PATCH 20/24] try to color red --- .../RecordCollectionController.groovy | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index 214496c..ed20cb4 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -1,5 +1,6 @@ package uk.co.metadataconsulting.sentinel +import builders.dsl.spreadsheet.api.Color import builders.dsl.spreadsheet.builder.poi.PoiSpreadsheetBuilder import grails.config.Config import grails.core.support.GrailsConfigurationAware @@ -145,12 +146,22 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon if ( view.rows ) { for (RecordCollectionExportRowView rowView : findAllValid(view.rows) ) { row { - for (String val : rowView.toList()) { - cell { - value val + for ( RecordPortion recordPortion : rowView.recordPortionList) { + for (String val : recordPortion.toList()) { + cell { +// if (recordPortion.status == ValidationStatus.INVALID) { +// style { +// font { +// color red +// } +// } +// } + value val + } } } - } } + } + } } } sheet(messageSource.getMessage("export.invalid".toString(), [] as Object[], 'Invalid', request.locale)) { @@ -177,9 +188,18 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon if ( view.rows ) { for (RecordCollectionExportRowView rowView : findAllInvalid(view.rows) ) { row { - for (String val : rowView.toList()) { - cell { - value val + for ( RecordPortion recordPortion : rowView.recordPortionList) { + for (String val : recordPortion.toList()) { + cell { +// if (recordPortion.status == ValidationStatus.INVALID) { +// style { +// font { +// color red +// } +// } +// } + value val + } } } } @@ -210,9 +230,18 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon if ( view.rows ) { for (RecordCollectionExportRowView rowView : findAllNotValidated(view.rows) ) { row { - for (String val : rowView.toList()) { - cell { - value val + for ( RecordPortion recordPortion : rowView.recordPortionList) { + for (String val : recordPortion.toList()) { + cell { +// if (recordPortion.status == ValidationStatus.INVALID) { +// style { +// font { +// color red +// } +// } +// } + value val + } } } } From 7aa0ea7b64b0ee6ee992030ebab66a9dffc56847 Mon Sep 17 00:00:00 2001 From: David Milward Date: Sun, 1 Jul 2018 19:41:10 +0100 Subject: [PATCH 21/24] Excel Output - Updated output into 3 sections - Valid/Invalid and not yet validated. Fixed Sorting problems Fixed display issues - except for colour issue --- .../RecordCollectionController.groovy | 200 ++++++++++++++---- .../sentinel/RecordService.groovy | 4 +- .../RecordCollectionExportService.groovy | 4 +- grails-app/views/record/index.gsp | 2 +- 4 files changed, 160 insertions(+), 50 deletions(-) diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index ed20cb4..335eee3 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -5,6 +5,7 @@ import builders.dsl.spreadsheet.builder.poi.PoiSpreadsheetBuilder import grails.config.Config import grails.core.support.GrailsConfigurationAware import groovy.util.logging.Slf4j +import org.apache.poi.xssf.usermodel.XSSFColor import org.springframework.context.MessageSource import uk.co.metadataconsulting.sentinel.export.ExportRecordCollectionCommand import uk.co.metadataconsulting.sentinel.export.ExportService @@ -12,6 +13,9 @@ import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportRowView import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportService import uk.co.metadataconsulting.sentinel.export.RecordCollectionExportView import uk.co.metadataconsulting.sentinel.modelcatalogue.ValidationRules + +import java.awt.Color + import static org.springframework.http.HttpStatus.OK import uk.co.metadataconsulting.sentinel.export.ExportFormat @@ -100,10 +104,59 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon response.status = OK.value() RecordCorrectnessDropdown recordCorrectnessDropdown = RecordCorrectnessDropdown.ALL - RecordCollectionExportView view = recordCollectionExportService.export(cmd.recordCollectionId, recordCorrectnessDropdown) + RecordCollectionExportView viewAll = recordCollectionExportService.export(cmd.recordCollectionId, recordCorrectnessDropdown) + RecordCollectionExportView view2 = recordCollectionExportService.export(cmd.recordCollectionId, recordCorrectnessDropdown) + RecordCollectionExportView view3 = recordCollectionExportService.export(cmd.recordCollectionId, recordCorrectnessDropdown) def outs = response.outputStream - List headers = view.headers - List portionsHeaders = view.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + + viewAll.rows = findAllValid(viewAll.rows) + view2.rows = findAllInvalid(view2.rows) + view3.rows = findAllNotValidated(view3.rows) + + List headers = [] + + //Ensure that all portions are sorted + viewAll.rows.each { RecordCollectionExportRowView rowView -> + List dataItemList = rowView.recordPortionList + dataItemList.sort { + it.header + } + } + + //Ensure that all portions are sorted + view2.rows.each { RecordCollectionExportRowView rowView -> + List dataItemList = rowView.recordPortionList + dataItemList.sort { + it.header + } + } + + //Ensure that all portions are sorted + view3.rows.each { RecordCollectionExportRowView rowView -> + List dataItemList = rowView.recordPortionList + dataItemList.sort { + it.header + } + } + + RecordCollectionExportRowView firstRow1 = viewAll.rows.first() + viewAll.headers = firstRow1.recordPortionList*.header + List headers1 = viewAll.headers + List portionsHeaders1 = viewAll.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + + RecordCollectionExportRowView firstRow2 = view2.rows.first() + view2.headers = firstRow2.recordPortionList*.header + List headers2 = view2.headers + List portionsHeaders2 = view2.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + + RecordCollectionExportRowView firstRow3 = view3.rows.first() + view3.headers = firstRow3.recordPortionList*.header + List headers3 = view3.headers + List portionsHeaders3 = view3.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + + + + String filename = "${filenameprefix}.${exportService.fileExtensionForFormat(cmd.format)}" response.setHeader "Content-disposition", "attachment; filename=${filename}" response.contentType = exportService.mimeTypeForFormat(cmd.format) @@ -112,10 +165,10 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon // TODO remove this line? headers = headers.collect { [it, it, it, it, it, it] }.flatten() outs << "${headers.join(separator)}\n" - if ( view.rows ) { + if ( viewAll.rows ) { String line = "${portionsHeaders.join(separator)}\n" outs << line - view.rows.each { RecordCollectionExportRowView row -> + viewAll.rows.each { RecordCollectionExportRowView row -> outs << "${row.toCsv(separator)}\n" } } @@ -124,16 +177,16 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon PoiSpreadsheetBuilder.create(outs).build { sheet(messageSource.getMessage("export.valid".toString(), [] as Object[], 'Valid', request.locale)) { row { - for ( String header : headers ) { + for ( String header : headers1 ) { cell { value header colspan RecordPortion.toHeaderList().size() } } } - if ( portionsHeaders ) { + if ( portionsHeaders1 ) { row { - for ( String recordPortionHeader : portionsHeaders ) { + for ( String recordPortionHeader : portionsHeaders1 ) { cell { value messageSource.getMessage("export.header.${recordPortionHeader}".toString(), [] as Object[], @@ -143,19 +196,13 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } } } - if ( view.rows ) { - for (RecordCollectionExportRowView rowView : findAllValid(view.rows) ) { - row { + if ( viewAll.rows ) { + //viewAll.rows.each{List rowView -> + for (RecordCollectionExportRowView rowView: viewAll.rows){ + row { for ( RecordPortion recordPortion : rowView.recordPortionList) { for (String val : recordPortion.toList()) { cell { -// if (recordPortion.status == ValidationStatus.INVALID) { -// style { -// font { -// color red -// } -// } -// } value val } } @@ -166,16 +213,16 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } sheet(messageSource.getMessage("export.invalid".toString(), [] as Object[], 'Invalid', request.locale)) { row { - for ( String header : headers ) { + for ( String header : headers2 ) { cell { value header colspan RecordPortion.toHeaderList().size() } } } - if ( portionsHeaders ) { + if ( portionsHeaders1 ) { row { - for ( String recordPortionHeader : portionsHeaders ) { + for ( String recordPortionHeader : portionsHeaders2 ) { cell { value messageSource.getMessage("export.header.${recordPortionHeader}".toString(), [] as Object[], @@ -185,20 +232,17 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } } } - if ( view.rows ) { - for (RecordCollectionExportRowView rowView : findAllInvalid(view.rows) ) { + if ( view2.rows ) { + for (RecordCollectionExportRowView rowView2 : view2.rows ) { row { - for ( RecordPortion recordPortion : rowView.recordPortionList) { + for ( RecordPortion recordPortion : rowView2.recordPortionList) { for (String val : recordPortion.toList()) { cell { -// if (recordPortion.status == ValidationStatus.INVALID) { -// style { -// font { -// color red -// } -// } -// } value val + style{ + + } + } } } @@ -208,16 +252,16 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } sheet(messageSource.getMessage("export.notValidated".toString(), [] as Object[], 'Not Validated', request.locale)) { row { - for ( String header : headers ) { + for ( String header : headers3 ) { cell { value header colspan RecordPortion.toHeaderList().size() } } } - if ( portionsHeaders ) { + if ( portionsHeaders3 ) { row { - for ( String recordPortionHeader : portionsHeaders ) { + for ( String recordPortionHeader : portionsHeaders3) { cell { value messageSource.getMessage("export.header.${recordPortionHeader}".toString(), [] as Object[], @@ -227,13 +271,13 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } } } - if ( view.rows ) { - for (RecordCollectionExportRowView rowView : findAllNotValidated(view.rows) ) { + if ( view3.rows ) { + for (RecordCollectionExportRowView rowView3 : view3.rows ) { row { - for ( RecordPortion recordPortion : rowView.recordPortionList) { + for ( RecordPortion recordPortion : rowView3.recordPortionList) { for (String val : recordPortion.toList()) { cell { -// if (recordPortion.status == ValidationStatus.INVALID) { +// if (isInValidPortionFunction(recordPortion)) { // style { // font { // color red @@ -257,23 +301,87 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } private List findAllValid(List all) { - all.findAll { RecordCollectionExportRowView row -> - !row.recordPortionList.any { it.status == ValidationStatus.INVALID } && row.recordPortionList.any { it.status == ValidationStatus.VALID } + List newList = all.each{ RecordCollectionExportRowView row -> + + def workingRecordPortionList = row.recordPortionList + List validDataItemList = [] + workingRecordPortionList.each{ portion -> + boolean bResult = isValidPortionFunction( portion) + println "Is Valid:" + bResult + ":" + portion.toString() + if(bResult){ + validDataItemList.add(portion) + println "added portion" + }else{ + validDataItemList.add(new RecordPortion(portion.header)) + println "empty portion added " + } + } + row.recordPortionList = validDataItemList } + return newList } - private List findAllInvalid(List all) { - all.findAll { RecordCollectionExportRowView row -> - row.recordPortionList.any { it.status == ValidationStatus.INVALID } + List newList = all.each{ RecordCollectionExportRowView row -> + + def workingRecordPortionList = row.recordPortionList + List validDataItemList = [] + workingRecordPortionList.each{ portion -> + boolean bResult = isInValidPortionFunction(portion) + println "Is In Valid:" + bResult + ":" + portion.toString() + + if(bResult){ + validDataItemList.add(portion) + println "added portion" + }else{ + validDataItemList.add(new RecordPortion(portion.header)) + println "empty portion added " + } + } + row.recordPortionList = validDataItemList } + return newList } - private List findAllNotValidated(List all) { - all.findAll { RecordCollectionExportRowView row -> - row.recordPortionList.every { it.status == ValidationStatus.NOT_VALIDATED } + List newList = all.each{ RecordCollectionExportRowView row -> + + def workingRecordPortionList = row.recordPortionList + List validDataItemList = [] + workingRecordPortionList.each{ portion -> + boolean bResult = isNotYetValidatedPortionFunction(portion) + println "Is Not Validated:" + bResult + ":" + portion.toString() + if(bResult){ + validDataItemList.add(portion) + println "added portion" + }else{ + validDataItemList.add(new RecordPortion(portion.header)) + println "empty portion added " + } + } + row.recordPortionList = validDataItemList } + return newList + } + + + private boolean isValidPortionFunction(RecordPortion portion){ + ValidationStatus vStatus = portion.status as ValidationStatus + boolean bstatus = (vStatus == ValidationStatus.VALID) + return bstatus } + private boolean isInValidPortionFunction(RecordPortion portion){ + ValidationStatus vStatus = portion.status as ValidationStatus + boolean bstatus = (vStatus == ValidationStatus.INVALID) + return bstatus + } + + private boolean isNotYetValidatedPortionFunction(RecordPortion portion){ + ValidationStatus vStatus = portion.status as ValidationStatus + boolean bstatus = (vStatus == ValidationStatus.NOT_VALIDATED) + return bstatus + } + + def importCsv() { [:] } diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy index f927042..448e902 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy @@ -64,9 +64,11 @@ class RecordService { new RecordViewModel(id: id, valid: !invalidRecordIds.contains(id)) } case RecordCorrectnessDropdown.VALID: - return findAllIdsByRecordCollectionId(recordCollectionId, correctness, paginationQuery).collect { + List recordCollection = findAllIdsByRecordCollectionId(recordCollectionId, correctness, paginationQuery) + List validViews = recordCollection.collect { new RecordViewModel(id: it, valid: true) } + return validViews case RecordCorrectnessDropdown.INVALID: return findAllIdsByRecordCollectionId(recordCollectionId, correctness, paginationQuery).collect { new RecordViewModel(id: it, valid: false) diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportService.groovy index 451436b..f95a724 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportService.groovy @@ -18,8 +18,8 @@ class RecordCollectionExportService { RecordGormService recordGormService - RecordCollectionExportView export(Long recordCollectionId, RecordCorrectnessDropdown correctnes = RecordCorrectnessDropdown.VALID) { - List recordIds = recordService.findAllIdsByRecordCollectionId(recordCollectionId, correctnes, null) + RecordCollectionExportView export(Long recordCollectionId, RecordCorrectnessDropdown correctness = RecordCorrectnessDropdown.VALID) { + List recordIds = recordService.findAllIdsByRecordCollectionId(recordCollectionId, correctness, null) List recordGormEntityList = recordGormService.findAllByIds(recordIds) List rows = [] List headers diff --git a/grails-app/views/record/index.gsp b/grails-app/views/record/index.gsp index b0440aa..2e84ad9 100644 --- a/grails-app/views/record/index.gsp +++ b/grails-app/views/record/index.gsp @@ -20,7 +20,7 @@
  • - + From 9b6945baed0d913d0da537b6a06ff4e66df4520e Mon Sep 17 00:00:00 2001 From: David Milward Date: Mon, 2 Jul 2018 23:57:49 +0100 Subject: [PATCH 22/24] Added excel export for minimal output --- .../RecordCollectionController.groovy | 279 ++++++++++++++++-- 1 file changed, 259 insertions(+), 20 deletions(-) diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index 335eee3..9e32e4b 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -60,7 +60,7 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon @Override void setConfiguration(Config co) { - separator = co.getProperty('export.csv.separator', String, ';') + separator = co.getProperty('export.csv.separator', String, ',') defaultPaginationMax = co.getProperty('sentinel.pagination.max', Integer, 25) defaultPaginationOffset = co.getProperty('sentinel.pagination.offset', Integer, 0) } @@ -88,6 +88,7 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon ] } + def export(ExportRecordCollectionCommand cmd) { if ( cmd.hasErrors() ) { @@ -109,9 +110,211 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon RecordCollectionExportView view3 = recordCollectionExportService.export(cmd.recordCollectionId, recordCorrectnessDropdown) def outs = response.outputStream - viewAll.rows = findAllValid(viewAll.rows) - view2.rows = findAllInvalid(view2.rows) - view3.rows = findAllNotValidated(view3.rows) + // viewAll.rows = findAllValid(viewAll.rows, false) + view2.rows = findAllValidRows(viewAll.rows ) + view3.rows = findAllInValidRows(viewAll.rows) + + //Ensure that all portions are sorted + viewAll.rows.each { RecordCollectionExportRowView rowView -> + List dataItemList = rowView.recordPortionList + dataItemList.sort { + it.header + } + } + + //Ensure that all portions are sorted + view2.rows.each { RecordCollectionExportRowView rowView -> + List dataItemList = rowView.recordPortionList + dataItemList.sort { + it.header + } + } + + //Ensure that all portions are sorted + view3.rows.each { RecordCollectionExportRowView rowView -> + List dataItemList = rowView.recordPortionList + dataItemList.sort { + it.header + } + } + + RecordCollectionExportRowView firstRow1 = viewAll.rows.first() + viewAll.headers = firstRow1.recordPortionList*.header + List headers1 = viewAll.headers + List portionsHeaders1 = viewAll.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + + RecordCollectionExportRowView firstRow2 = view2.rows.first() + view2.headers = firstRow2.recordPortionList*.header + List headers2 = view2.headers + List portionsHeaders2 = view2.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + + RecordCollectionExportRowView firstRow3 = view3.rows.first() + view3.headers = firstRow3.recordPortionList*.header + List headers3 = view3.headers + List portionsHeaders3 = view3.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + + + + + String filename = "${filenameprefix}.${exportService.fileExtensionForFormat(cmd.format)}" + response.setHeader "Content-disposition", "attachment; filename=${filename}" + response.contentType = exportService.mimeTypeForFormat(cmd.format) + switch (cmd.format) { + case ExportFormat.CSV: + // TODO remove this line? + headers = headers1.collect { [it] }.flatten() + outs << "${headers1.join(separator)}\n" + if ( viewAll.rows ) { + String line = "${portionsHeaders1.join(separator)}\n" + outs << line + viewAll.rows.each { RecordCollectionExportRowView row -> + println row. + outs << "${row.toCsv(separator)}\n" + } + } + break + case ExportFormat.XLSX: + PoiSpreadsheetBuilder.create(outs).build { + sheet(messageSource.getMessage("export.all".toString(), [] as Object[], 'All', request.locale)) { + row { + for (String header : headers1) { + cell { + value header + //colspan RecordPortion.toHeaderList().size() + } + } + } + + if (viewAll.rows) { + //viewAll.rows.each{List rowView -> + for (RecordCollectionExportRowView rowView : viewAll.rows) { + row { + for (RecordPortion recordPortion : rowView.recordPortionList) { + cell { + if (isValidPortionFunction(recordPortion)) { + style { + font { + color green + } + } + } + if (isInValidPortionFunction(recordPortion)) { + style { + font { + color red + } + } + } + value recordPortion.value + } + } + } + } + } + } + sheet(messageSource.getMessage("export.valid".toString(), [] as Object[], 'Valid', request.locale)) { + row { + for (String header : headers2) { + cell { + value header + } + } + } + + if (view2.rows) { + //viewAll.rows.each{List rowView -> + for (RecordCollectionExportRowView rowView : view2.rows) { + row { + for (RecordPortion recordPortion : rowView.recordPortionList) { + cell { + if (isValidPortionFunction(recordPortion)) { + style { + font { + color green + } + } + } + if (isInValidPortionFunction(recordPortion)) { + style { + font { + color red + } + } + } + value recordPortion.value + } + } + } + } + } + } + sheet(messageSource.getMessage("export.invalid".toString(), [] as Object[], 'Invalid', request.locale)) { + row { + for (String header : headers3) { + cell { + value header + //colspan RecordPortion.toHeaderList().size() + } + } + } + if (view3.rows) { + //viewAll.rows.each{List rowView -> + for (RecordCollectionExportRowView rowView : view3.rows) { + row { + for (RecordPortion recordPortion : rowView.recordPortionList) { + cell { + if (isValidPortionFunction(recordPortion)) { + style { + font { + color green + } + } + } + if (isInValidPortionFunction(recordPortion)) { + style { + font { + color red + } + } + } + value recordPortion.value + } + } + } + } + } + } + } + break + } + outs.flush() + outs.close() + } + + def exportAll(ExportRecordCollectionCommand cmd) { + + if ( cmd.hasErrors() ) { + // TODO Maybe display an error message with flash.error and redirect to HomePage + return + } + + RecordCollectionGormEntity collectionGormEntity = recordCollectionGormService.find(cmd.recordCollectionId) + if ( !collectionGormEntity ) { + // TODO Maybe display an error message with flash.error and redirect to HomePage + return + } + final String filenameprefix = collectionGormEntity.datasetName + response.status = OK.value() + + RecordCorrectnessDropdown recordCorrectnessDropdown = RecordCorrectnessDropdown.ALL + RecordCollectionExportView viewAll = recordCollectionExportService.export(cmd.recordCollectionId, recordCorrectnessDropdown) + RecordCollectionExportView view2 = recordCollectionExportService.export(cmd.recordCollectionId, recordCorrectnessDropdown) + RecordCollectionExportView view3 = recordCollectionExportService.export(cmd.recordCollectionId, recordCorrectnessDropdown) + def outs = response.outputStream + + viewAll.rows = findAllValid(viewAll.rows, true) + view2.rows = findAllInvalid(view2.rows, true) + view3.rows = findAllNotValidated(view3.rows, true) List headers = [] @@ -277,13 +480,13 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon for ( RecordPortion recordPortion : rowView3.recordPortionList) { for (String val : recordPortion.toList()) { cell { -// if (isInValidPortionFunction(recordPortion)) { -// style { -// font { -// color red -// } -// } -// } + + style { + font { + color red + } + } + value val } } @@ -300,7 +503,36 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon outs.close() } - private List findAllValid(List all) { + private List findAllValidRows(List all ) { + List newList = [] + all.each{ RecordCollectionExportRowView row -> + int cntr = 0 + row.recordPortionList.each{ portion -> + boolean bResult = isValidPortionFunction( portion) + if(bResult){cntr++} + } + if(cntr > 0){ + newList.add(row) + } + } + return newList + } + private List findAllInValidRows(List all ) { + List newList = [] + all.each{ RecordCollectionExportRowView row -> + int cntr = 0 + row.recordPortionList.each{ portion -> + boolean bResult = isInValidPortionFunction( portion) + if(bResult){cntr++} + } + if(cntr > 0){ + newList.add(row) + } + } + return newList + } + + private List findAllValid(List all, boolean addRows) { List newList = all.each{ RecordCollectionExportRowView row -> def workingRecordPortionList = row.recordPortionList @@ -312,15 +544,18 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon validDataItemList.add(portion) println "added portion" }else{ - validDataItemList.add(new RecordPortion(portion.header)) - println "empty portion added " + if(addRows){ + validDataItemList.add(new RecordPortion(portion.header)) + println "empty portion added " + } + } } row.recordPortionList = validDataItemList } return newList } - private List findAllInvalid(List all) { + private List findAllInvalid(List all, boolean addRows) { List newList = all.each{ RecordCollectionExportRowView row -> def workingRecordPortionList = row.recordPortionList @@ -333,15 +568,17 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon validDataItemList.add(portion) println "added portion" }else{ - validDataItemList.add(new RecordPortion(portion.header)) - println "empty portion added " + if(addRows){ + validDataItemList.add(new RecordPortion(portion.header)) + println "empty portion added " + } } } row.recordPortionList = validDataItemList } return newList } - private List findAllNotValidated(List all) { + private List findAllNotValidated(List all, boolean addRows) { List newList = all.each{ RecordCollectionExportRowView row -> def workingRecordPortionList = row.recordPortionList @@ -353,8 +590,10 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon validDataItemList.add(portion) println "added portion" }else{ - validDataItemList.add(new RecordPortion(portion.header)) - println "empty portion added " + if(addRows){ + validDataItemList.add(new RecordPortion(portion.header)) + println "empty portion added " + } } } row.recordPortionList = validDataItemList From 106b532588898945c886e1565b0801a30549308c Mon Sep 17 00:00:00 2001 From: David Milward Date: Tue, 3 Jul 2018 00:03:07 +0100 Subject: [PATCH 23/24] Update --- gradle.properties | 2 +- .../uk/co/metadataconsulting/sentinel/RecordService.groovy | 3 ++- .../sentinel/export/RecordCollectionExportRowView.groovy | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gradle.properties b/gradle.properties index 7987827..a165ded 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,7 +2,6 @@ jsonViewsVersion=1.2.7 webdriverBinariesGradleVersion=1.4 grailsWrapperVersion=1.0.0 -poiVersion=3.17 grailsVersion=3.3.6 hamcrestVersion=1.3 seleniumVersion=3.6.0 @@ -18,3 +17,4 @@ droolsVersion=7.5.0.Final chromedriverVersion=2.40 geckodriverVersion=0.21.0 spreadsheetBuilderVersion=1.0.5 +poiVersion=3.16 diff --git a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy index 448e902..11158aa 100644 --- a/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy +++ b/grails-app/services/uk/co/metadataconsulting/sentinel/RecordService.groovy @@ -116,7 +116,8 @@ class RecordService { return [] as List } DetachedCriteria query = queryInvalidRecords(recordCollectionId, invalidRecordIds) - query.id().list(paginationQuery.toMap()) as List + //query.id().list(paginationQuery.toMap()) as List + return (paginationQuery ? query.id().list(paginationQuery.toMap()) : query.id().list()) as List } Number countValidRecords(Long recordCollectionId) { diff --git a/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy index 2ac24b6..99fbf64 100644 --- a/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy +++ b/src/main/groovy/uk/co/metadataconsulting/sentinel/export/RecordCollectionExportRowView.groovy @@ -7,7 +7,7 @@ import uk.co.metadataconsulting.sentinel.RecordPortion class RecordCollectionExportRowView { List recordPortionList - String toCsv(String separator = ';') { + String toCsv(String separator = ',') { recordPortionList.collect { it.toCsv(separator) }.join(separator) } From 5f5d9b788e5fd215f40a5b8aea32d22bb463befd Mon Sep 17 00:00:00 2001 From: Adam Milward Date: Mon, 9 Jul 2018 16:38:35 +0100 Subject: [PATCH 24/24] bugfix to prevent crash when you haven't validated the records and export + design changes --- .../RecordCollectionController.groovy | 15 ++- grails-app/views/record/index.gsp | 122 ++++++++++++------ 2 files changed, 91 insertions(+), 46 deletions(-) diff --git a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy index 9e32e4b..49dad61 100644 --- a/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy +++ b/grails-app/controllers/uk/co/metadataconsulting/sentinel/RecordCollectionController.groovy @@ -138,20 +138,21 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } } - RecordCollectionExportRowView firstRow1 = viewAll.rows.first() + RecordCollectionExportRowView firstRow1 = (viewAll?.rows)?viewAll.rows.first():[] viewAll.headers = firstRow1.recordPortionList*.header List headers1 = viewAll.headers - List portionsHeaders1 = viewAll.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() - RecordCollectionExportRowView firstRow2 = view2.rows.first() + if(viewAll?.rows) List portionsHeaders1 = viewAll.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + + RecordCollectionExportRowView firstRow2 = (view2?.rows)?view2.rows.first():[] view2.headers = firstRow2.recordPortionList*.header List headers2 = view2.headers - List portionsHeaders2 = view2.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + if(view2?.rows) List portionsHeaders2 = view2.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() - RecordCollectionExportRowView firstRow3 = view3.rows.first() + RecordCollectionExportRowView firstRow3 = (view2?.rows)?view3.rows.first():[] view3.headers = firstRow3.recordPortionList*.header List headers3 = view3.headers - List portionsHeaders3 = view3.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() + if(view3?.rows) List portionsHeaders3 = view3.rows.first().recordPortionList.collect { RecordPortion.toHeaderList() }.flatten() @@ -712,4 +713,4 @@ class RecordCollectionController implements ValidateableErrorsMessage, GrailsCon } null } -} \ No newline at end of file +} diff --git a/grails-app/views/record/index.gsp b/grails-app/views/record/index.gsp index 2e84ad9..27c2133 100644 --- a/grails-app/views/record/index.gsp +++ b/grails-app/views/record/index.gsp @@ -7,59 +7,102 @@ -