From 4fb5fb906d425f577e501f914450699597862b67 Mon Sep 17 00:00:00 2001 From: Marten Deinum Date: Wed, 18 Sep 2024 16:46:35 +0200 Subject: [PATCH] Update documentation Document on how to use the BeanWrapperRowMapper with the StreamingXlsxItemReader as that requires a bit more configuration due to the streaming nature. Next to the documentation also added a test case for this to show how to configure it as well. Closes: #115 --- spring-batch-excel/README.adoc | 91 ++++++++++++++++++- .../streaming/StreamingXlsxMappingTests.java | 62 +++++++++++++ 2 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 spring-batch-excel/src/test/java/org/springframework/batch/extensions/excel/streaming/StreamingXlsxMappingTests.java diff --git a/spring-batch-excel/README.adoc b/spring-batch-excel/README.adoc index eea472ac..42671c0f 100644 --- a/spring-batch-excel/README.adoc +++ b/spring-batch-excel/README.adoc @@ -128,15 +128,98 @@ Uses a `BeanWrapper` to convert a given row into an object. Uses the column name ---- +or the same in Java configuration. + +[source,java] +---- +@Bean +public PoiItemReader excelReader(BeanWrapperRowMapper rowMapper) { + var excelReader = new PoiItemReader(); + excelReader.setResource(new FileSystemResouce("file:/path/to/your/excel/file")); + excelReader.setRowMapper(rowMapper); + return excelReader; +} + +@Bean +public BeanWrapperRowMapper rowMapper() { + var rowMapper = new BeanWrapperRowMapper(); + rowMapper.setTargetType(Player.class); + return rowMapper; +} +---- + +NOTE: When using the `BeanWrapperRowMapper` with the `StreamingXlsxItemReader` it is required to use the `StaticColumnNameExtractor` to provide the column names for mapping purposes. The reason for this is that we cannot read a specific row while streaming the results. + +[source,xml] +---- + + + + + + + + + + + + + + + + + +---- + +And the same in Java Configuration. + +[source,java] +---- +@Bean +public StreamingXlsxItemReader excelReader(BeanWrapperRowMapper rowMapper) { + var columns = new String[] {"id", "position", "lastName", "firstName", "birthYear", "debutYear","comment"}; + var columnNameExtractor = new StaticColumnNameExtractor(columns); + var rowSetFactory = new DefaultRowSetFactory(); + rowSetFactory.setColumnNameExtractor(columnNameExtractor); + + var excelReader = new StreamingXlsxItemReader(); + excelReader.setResource(new FileSystemResouce("file:/path/to/your/excel/file")); + excelReader.setRowMapper(rowMapper); + excelReader.set + return excelReader; +} + +@Bean +public BeanWrapperRowMapper rowMapper() { + var rowMapper = new BeanWrapperRowMapper(); + rowMapper.setTargetType(Player.class); + return rowMapper; +} +---- + == Frequently Asked Questions === Not able to open large Excel When opening large Excel files or Excel files with large amounts of data in a single cell it might fail with an error -``` +[source] +---- "Unexpected error Tried to allocate an array of length 162,386,364, but the maximum length for this record type is 100,000,000. If the file is not corrupt or large, please open an issue on bugzilla to request increasing the maximum allowable size for this record type. As a temporary workaround, consider setting a higher override value with IOUtils.setByteArrayMaxOverride()" -``` +---- -This is due to the maximum lenght for certain datatypes is limited. To prevent this from happening you can use the `IOUtils.setByteArrayMaxOverride()` method to increase the allowed size. It is however important that this is set before anything POI related has been processed/configured. +This is due to the maximum lenght for certain datatypes is limited. To prevent this from happening you can use the `IOUtils.setByteArrayMaxOverride()` method to increase the allowed size. It is however important that this is set _before_ anything POI related has been processed/configured. -Ideally, when using Spring Boot, you can set this before launching the application or by putting this in a `static {}` initializer block of the Spring Batch job configuration. +Ideally, when using Spring Boot, you can set this before launching the application or by putting this in a `static {}` initializer block of the Spring Batch job configuration. + +[source,java] +---- +import org.apache.poi.util.IOUtils; +@SpringBootApplication +public class MyBatchApplication { + + public static void main(String[] args) { + IOUtils.setByteArrayMaxOverride(Integer.MAX_VALUE); + SpringApplication.run(MyBatchApplication.class, args); + } +} +---- diff --git a/spring-batch-excel/src/test/java/org/springframework/batch/extensions/excel/streaming/StreamingXlsxMappingTests.java b/spring-batch-excel/src/test/java/org/springframework/batch/extensions/excel/streaming/StreamingXlsxMappingTests.java new file mode 100644 index 00000000..3751c18d --- /dev/null +++ b/spring-batch-excel/src/test/java/org/springframework/batch/extensions/excel/streaming/StreamingXlsxMappingTests.java @@ -0,0 +1,62 @@ +/* + * Copyright 2002-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.excel.streaming; + +import java.util.Locale; + +import org.junit.jupiter.api.Test; + +import org.springframework.batch.extensions.excel.Player; +import org.springframework.batch.extensions.excel.ReflectionTestUtils; +import org.springframework.batch.extensions.excel.mapping.BeanWrapperRowMapper; +import org.springframework.batch.extensions.excel.support.rowset.DefaultRowSetFactory; +import org.springframework.batch.extensions.excel.support.rowset.StaticColumnNameExtractor; +import org.springframework.batch.item.ExecutionContext; +import org.springframework.core.io.ClassPathResource; + +import static org.assertj.core.api.Assertions.assertThat; + +class StreamingXlsxMappingTests { + + @Test + void readAndMapRowsUsingRowMapper() throws Exception { + var columns = new String[] {"id", "position", "lastName", "firstName", "birthYear", "debutYear", "comment"}; + var rowSetFactory = new DefaultRowSetFactory(); + rowSetFactory.setColumnNameExtractor(new StaticColumnNameExtractor(columns)); + + var mapper = new BeanWrapperRowMapper(); + mapper.setTargetType(Player.class); + + var reader = new StreamingXlsxItemReader(); + reader.setResource(new ClassPathResource("player.xlsx")); + reader.setRowSetFactory(rowSetFactory); + reader.setRowMapper(mapper); + reader.setLinesToSkip(1); // Skip header + reader.setUserLocale(Locale.US); // Use a Locale to not be dependent on environment + reader.afterPropertiesSet(); + + reader.open(new ExecutionContext()); + Player row; + do { + row = reader.read(); + } + while (row != null); + + Integer readCount = (Integer) ReflectionTestUtils.getField(reader, "currentItemCount"); + assertThat(readCount).isEqualTo(4321); + } +}