CSVMessageConverter.java

/*
 * Copyright 2019 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.gringlobal.spring;

import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.JsonViews;
import org.gringlobal.api.v1.FilteredPage;
import org.gringlobal.soap.Datatable;
import org.gringlobal.soap.Datatable.Row;
import org.springframework.data.domain.Page;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpInputMessage;
import org.springframework.http.HttpOutputMessage;
import org.springframework.http.MediaType;
import org.springframework.http.converter.AbstractHttpMessageConverter;
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.http.converter.HttpMessageNotWritableException;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.collect.Sets;
import com.opencsv.CSVWriter;

/**
 * @author Maxym Borodenko
 */
@Slf4j
public class CSVMessageConverter<T> extends AbstractHttpMessageConverter<T> {

	public static final String DEFAULT_FIELD_SELECTOR = "select";
	public static final String DEFAULT_EXCLUDE_FIELD = "exclude";

	public static final String TEXT_CSV_VALUE = "text/csv";
	public static final MediaType TEXT_CSV = MediaType.valueOf(TEXT_CSV_VALUE);

	public static final String TEXT_TSV_VALUE = "text/tsv";
	public static final MediaType TEXT_TSV = MediaType.valueOf(TEXT_TSV_VALUE);

	private final ObjectMapper mapper;

	// allow configuration of the fields name
	private String fieldsParam = DEFAULT_FIELD_SELECTOR;
	private String excludeFieldsParam = DEFAULT_EXCLUDE_FIELD;

	private final Set<Pattern> IGNORED_HEADERS = Sets.newHashSet(
		Pattern.compile("_permissions\\..+$", Pattern.MULTILINE),
		Pattern.compile("_class$", Pattern.MULTILINE),
		Pattern.compile("\\.id$", Pattern.MULTILINE),
		Pattern.compile("\\.version$", Pattern.MULTILINE)
	);

	public void setFieldsParam(final String fieldsParam) {
		this.fieldsParam = fieldsParam;
	}

	public void setExcludeFieldsParam(String excludeFieldsParam) {
		this.excludeFieldsParam = excludeFieldsParam;
	}

	public CSVMessageConverter(final ObjectMapper jsonObjectMapper) {
		super(TEXT_CSV, new MediaType("text", "*+csv"), TEXT_TSV);
		this.mapper = jsonObjectMapper;
	}

	@Override
	public boolean canWrite(final Class<?> clazz, final MediaType mediaType) {
		return mediaType != null && (mediaType.isCompatibleWith(TEXT_CSV) || mediaType.isCompatibleWith(TEXT_TSV));
	}

	@Override
	protected boolean supports(final Class<?> clazz) {
		return Page.class.isAssignableFrom(clazz) || Collection.class.isAssignableFrom(clazz);
	}

	@Override
	protected T readInternal(final Class<? extends T> clazz, final HttpInputMessage inputMessage) throws IOException, HttpMessageNotReadableException {
		throw new UnsupportedOperationException();
	}

	@Override
	protected void addDefaultHeaders(final HttpHeaders headers, final T t, final MediaType contentType) throws IOException {
		super.addDefaultHeaders(headers, t, contentType);

		if (Page.class.isAssignableFrom(t.getClass())) {
			final Page<?> page = (Page<?>) t;
			headers.add("Pagination-Page", String.valueOf(page.getNumber()));
			headers.add("Pagination-Size", String.valueOf(page.getSize()));
			headers.add("Pagination-Elements", String.valueOf(page.getNumberOfElements()));
			headers.add("Pagination-Total", String.valueOf(page.getTotalElements()));
		}

		if (Datatable.class.isAssignableFrom(t.getClass())) {
			final Datatable dt = (Datatable) t;
			headers.add("Pagination-Offset", String.valueOf(dt.getOffset()));
			headers.add("Pagination-Limit", String.valueOf(dt.getLimit()));
		}

		if (FilteredPage.class.isAssignableFrom(t.getClass())) {
			var filteredPage = (FilteredPage<?,?>) t;
			Page<?> page = filteredPage.page;
			if (StringUtils.isNotBlank(filteredPage.filterCode)) {
				headers.add("Pagination-URL", String.valueOf(((FilteredPage<?,?>) t).filterCode));
			}
			headers.add("Pagination-Page", String.valueOf(page.getNumber()));
			headers.add("Pagination-Size", String.valueOf(page.getSize()));
			headers.add("Pagination-Elements", String.valueOf(page.getNumberOfElements()));
			headers.add("Pagination-Total", String.valueOf(page.getTotalElements()));
		}
	}

	/**
	 * Method converts to CSV format and writes data to the output message
	 */
	@Override
	protected void writeInternal(final T t, final HttpOutputMessage outputMessage) throws IOException, HttpMessageNotWritableException {
		if (t == null) {
			return;
		}

		final Map<String, String[]> allParams = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest().getParameterMap();
		// is there a fields parameter in request
		List<String> includeFields = Arrays.asList(allParams.getOrDefault(fieldsParam, ArrayUtils.EMPTY_STRING_ARRAY));
		List<String> excludeFields = Arrays.asList(allParams.getOrDefault(excludeFieldsParam, ArrayUtils.EMPTY_STRING_ARRAY));
		if (includeFields.size() > 0 && includeFields.size() == excludeFields.size() && includeFields.containsAll(excludeFields) && excludeFields.containsAll(includeFields)) {
			return;
		}

		log.trace("Keeping only {}", includeFields);
		final ObjectWriter writer = mapper.writer().with(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")).withView(JsonViews.Public.class);

		final List<Map<String, Object>> records = new ArrayList<>();
		final Set<String> paths = new HashSet<>();
		final Set<String> initialPathsOfArrays = new HashSet<>();
		Map<String, Map<Number, ObjectNode>> cachedObjects = new HashMap<>();

		if (t instanceof Datatable) {
			writeDatatable((Datatable) t, outputMessage);
			return;

		} else if (t instanceof Iterable) {
			var content = Lists.newArrayList((Iterable<?>) t);
			// serialize a list of objects
			String s = writer.writeValueAsString(content);
			ArrayNode arrayNode = (ArrayNode) mapper.readTree(s);
			for (int i = 0; i < arrayNode.size(); i++) {
				Object sourceObj = content.get(i);
				doThings(sourceObj == null ? null : sourceObj.getClass(), arrayNode.get(i), records, paths, initialPathsOfArrays, includeFields, excludeFields, cachedObjects);
			}
		} else {
			// serialize single object
			String s = writer.writeValueAsString(t);
			doThings(t == null ? null : t.getClass(), mapper.readTree(s), records, paths, initialPathsOfArrays, includeFields, excludeFields, cachedObjects);
		}
		cachedObjects.clear();

		paths.removeAll(initialPathsOfArrays);

		try (CSVWriter csvWriter = new CSVWriter(new OutputStreamWriter(outputMessage.getBody(), StandardCharsets.UTF_8), '\t', '"', '\\', "\n")) {
			paths.removeIf(path -> IGNORED_HEADERS.stream().map(rx -> rx.matcher(path).find()).filter(found -> found).findFirst().orElse(false));

			// adding header to csv
			final String[] headers = paths.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
			Arrays.sort(headers);
			csvWriter.writeNext(headers, false);

			final List<Object> rowValues = new ArrayList<>();
			for (final Map<String, Object> row : records) {
				rowValues.clear();
				for (final String key : headers) {
					final Object v = row.get(key);
					rowValues.add(v == null ? null : v.toString());
				}
				csvWriter.writeNext(rowValues.toArray(ArrayUtils.EMPTY_STRING_ARRAY), false);
			}
		}
	}

	private void writeDatatable(final Datatable datatable, final HttpOutputMessage outputMessage) throws IOException {
		try (CSVWriter csvWriter = new CSVWriter(new OutputStreamWriter(outputMessage.getBody(), StandardCharsets.UTF_8), '\t', '"', '\\', "\n")) {
			// adding header to csv
			final List<String> headers = datatable.getColumns().stream().map(Datatable.Column::getName).collect(Collectors.toList());
			csvWriter.writeNext(headers.toArray(ArrayUtils.EMPTY_STRING_ARRAY), false);
			final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
			for (final Row row : datatable.getRows()) {
				csvWriter.writeNext(Arrays.stream(row.getData()).map(o -> {
					if (o == null) {
						return "";
					} else if (o instanceof Date || o instanceof Calendar) {
						return sdf.format(o);
					} else {
						return o.toString();
					}
				}).collect(Collectors.toList()).toArray(ArrayUtils.EMPTY_STRING_ARRAY), false);
			}
		}
	}

	private void doThings(Class<?> clazz, JsonNode jsonNode, List<Map<String, Object>> records, Set<String> paths, Set<String> initialPathsOfArrays,
		List<String> include, List<String> exclude, Map<String, Map<Number, ObjectNode>> cachedObjects) {

		JsonToFlatMapConverter flatMap = JsonToFlatMapConverter.fromJson(clazz, jsonNode, include, exclude, cachedObjects);
		initialPathsOfArrays.addAll(flatMap.getInitialPathsOfArrays());

		Map<String, Object> map = flatMap.getMap();
		paths.addAll(map.keySet());
		records.add(map);
	}
}