DataviewEndpoint.java

/*
 * Copyright 2019 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.gringlobal.soap;

import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import javax.persistence.PersistenceException;

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.time.StopWatch;
import org.gringlobal.api.exception.InvalidApiUsageException;
import org.gringlobal.compatibility.service.DataviewService;
import org.gringlobal.compatibility.service.impl.DataviewServiceImpl;
import org.gringlobal.soap.model.GetData;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.filter.Filters;
import org.jdom2.output.Format;
import org.jdom2.output.XMLOutputter;
import org.jdom2.xpath.XPathExpression;
import org.jdom2.xpath.XPathFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.ws.server.endpoint.annotation.Endpoint;
import org.springframework.ws.server.endpoint.annotation.PayloadRoot;
import org.springframework.ws.server.endpoint.annotation.RequestPayload;
import org.springframework.ws.server.endpoint.annotation.ResponsePayload;

@Endpoint
@Slf4j
public class DataviewEndpoint extends BaseEndpoint {

	public static final XPathExpression<Element> xpDatasetBefore;
	public static final XPathExpression<Element> xpDatasetAfter;
	public static final XPathExpression<Element> xpSchemaRoot;
	public static final XPathExpression<Element> xpDataviewSchema;

	@Autowired
	private DataviewService dataviewService;

	private final XMLOutputter xmlOutputter = new XMLOutputter(Format.getPrettyFormat());

	static {
		final XPathFactory xPathFactory = XPathFactory.instance();

		xpDatasetBefore = xPathFactory.compile("//diffgr:diffgram/diffgr:before/*", Filters.element(), null, GGXml.NsDIFFGR);
		xpDatasetAfter = xPathFactory.compile("//diffgr:diffgram/NewDataSet/*|//diffgr:diffgram/SecureDataDataSet/*", Filters.element(), null, GGXml.NsDIFFGR);
		// The schema root node that we use to inject ExceptionTable schema
		xpSchemaRoot = xPathFactory.compile("//xs:schema//xs:choice", Filters.element(), null, GGXml.NsSCHEMA);
		// This is the schema of the Dataview as provided by CT
		xpDataviewSchema = xPathFactory.compile("//xs:schema//xs:choice//xs:sequence", Filters.element(), null, GGXml.NsSCHEMA);
	}

	@PreAuthorize("hasAuthority('GROUP_CTUSERS') || hasAuthority('GROUP_ADMINS')")
	@PayloadRoot(namespace = GGXml.NAMESPACE_URI, localPart = "GetData")
	@ResponsePayload
	public Element getData(@RequestPayload final GetData request) throws SQLException {

		var xmlOutputter = new XMLOutputter();

		var stopWatch = StopWatch.createStarted();
		log.info("getData << {} request user={} offset={} limit={} params={}", request.getDataviewName(), request.getUserName(), request.getOffset(), request.getLimit(), request.getDelimitedParameterList());

		final Element response = new Element("GetDataResponse", GGXml.NAMESPACE_URI);

		try {
			final Map<String, String> parameters = DataviewServiceImpl.parseParameterList(request.getDelimitedParameterList());
	
			var datatable = dataviewService.getData(request.getDataviewName(), parameters, request.getOffset(), request.getLimit(), request.getOptions());
			stopWatch.split();
			log.trace("getData retrieved {}records at {}ms", datatable.getRows().size(), stopWatch.getSplitTime());
			stopWatch.unsplit();

			var dataviewDiffgram = datatable.toDiffgram("GetDataResult", GGXml.NAMESPACE_URI);
			stopWatch.split();
			log.trace("getData created diffgram for {}records at {}ms", datatable.getRows().size(), stopWatch.getSplitTime());
			stopWatch.unsplit();

			response.addContent(dataviewDiffgram);
			stopWatch.split();
			log.trace("getData added {}records to response at {}ms", datatable.getRows().size(), stopWatch.getSplitTime());

			log.info("getData >> {} response user={} rows={} throughput={}rps took={}ms", request.getDataviewName(), request.getUserName(), datatable.getRows().size(), Math.round((float) datatable.getRows().size() / stopWatch.getTime(TimeUnit.MILLISECONDS) * 1000f), stopWatch.getTime(TimeUnit.MILLISECONDS));
			stopWatch.unsplit();

		} catch (Throwable e) {
			if (log.isInfoEnabled() || e instanceof NullPointerException || e instanceof PersistenceException) {
				log.error("getData failed: {}", e.getMessage(), e);
			} else {
				log.error("getData failed: {}", e.getMessage());
			}

			// Handle exceptions the GRIN-Global way
			var exceptionDatatable = new Datatable();
			exceptionDatatable.addException(e);
			response.addContent(exceptionDatatable.toDiffgram("GetDataResult", GGXml.NAMESPACE_URI));
			if (log.isDebugEnabled()) {
				log.debug("getData error: {}", xmlOutputter.outputString(response));
			}
		}

		stopWatch.stop();
		log.info("getData {} took {}ms for user={}", request.getDataviewName(), stopWatch.getTime(TimeUnit.MILLISECONDS), request.getUserName());

		if (log.isTraceEnabled()) {
			log.trace("getData >> {} output: {}", request.getDataviewName(), xmlOutputter.outputString(response));
		}
		return response;
	}

	/**
	 * GRIN-Global SaveData endpoint.
	 * 
	 * CT sends rows in batches of whatever batch size is set to.
	 */
	@PreAuthorize("hasAuthority('GROUP_CTUSERS') || hasAuthority('GROUP_ADMINS')")
	@PayloadRoot(namespace = GGXml.NAMESPACE_URI, localPart = "SaveData")
	@ResponsePayload
	public Element saveData(@RequestPayload final Element request) throws SQLException {

		var stopWatch = StopWatch.createStarted();

		if (log.isTraceEnabled()) {
			log.trace("Received saveData request: {}", xmlOutputter.outputString(request));
		}

		final Element response = new Element("SaveDataResponse", GGXml.NAMESPACE_URI);

		boolean includeStackTraces = false;
		var suppressExceptions = request.getChild("suppressExceptions", request.getNamespace());
		if (suppressExceptions != null) {
			includeStackTraces = Objects.equals("false", suppressExceptions.getText());
		}
		log.debug("Suppress exceptions: {}", includeStackTraces);

		var sourceData = request.getChild("ds", request.getNamespace());
		if (sourceData == null) {
			throw new InvalidApiUsageException("Missing 'ds' element in SOAP request");
		}
		final Element magic = sourceData.clone();
		new Document(magic); // needed for XPath !

		try {
			// Try 1-by-1 just like GRIN-Global
			saveDataOneByOne(magic, includeStackTraces);

			stopWatch.split();
			log.info("saveDataOneByOne done and committed at {}ms", stopWatch.getSplitTime());
			stopWatch.unsplit();

			magic.detach();
			response.addContent(magic);

			stopWatch.split();
			log.debug("saveDataOneByOne content added at {}ms", stopWatch.getSplitTime());
			stopWatch.unsplit();

			if (log.isTraceEnabled()) {
				log.trace("Final one-by-one: {}", xmlOutputter.outputString(magic));
			}

		} catch (Throwable aBigError) {
			if (log.isInfoEnabled() || aBigError instanceof NullPointerException || aBigError instanceof ArrayIndexOutOfBoundsException) {
				log.warn("SaveData failed: {}", aBigError.getMessage(), aBigError); // Log NPEs!
			} else {
				log.warn("SaveData failed: {}", aBigError.getMessage());
			}

			// Handle exception the GRIN-Global way in case this dies completely
			var exceptionDatatable = new Datatable();
			exceptionDatatable.addException(aBigError);
			response.addContent(exceptionDatatable.toDiffgram("SaveDataResult", GGXml.NAMESPACE_URI));
		}

		stopWatch.stop();
		log.info("saveData took {}ms", stopWatch.getTime(TimeUnit.MILLISECONDS));

		return response;
	}

	/**
	 * Handle saveData row-by-row.
	 * @param includeStackTraces 
	 * @throws Throwable 
	 */
	private void saveDataOneByOne(Element magic, boolean includeStackTraces) {

		var before = xpDatasetBefore.evaluate(magic);
		var after = xpDatasetAfter.evaluate(magic);

		if (log.isDebugEnabled()) {
			log.debug("before: {}", xmlOutputter.outputString(before));
			log.debug("after: {}", xmlOutputter.outputString(after));
		}

		try {

			final List<Element> insertsAndUpdatesAndExceptions = dataviewService.saveDataOneByOne(before, after, includeStackTraces);

			magic.setName("SaveDataResult");

			// Add ExceptionTable to schema
			var responseSchemaRoot = xpSchemaRoot.evaluateFirst(magic);
			if (responseSchemaRoot != null) {
				responseSchemaRoot.addContent(Datatable.makeExceptionTableSchema()); // Add ExceptionTable
			} else {
				log.warn("Could not find {}", xpSchemaRoot);
			}

			expandDatatableSchemaForCT(magic); // Add extra columns

			final Element diffgramRoot = magic.getChild("diffgram", GGXml.NsDIFFGR);
			diffgramRoot.getChildren().clear();

			final Element datatable = new Element("SecureDataDataSet");
			diffgramRoot.addContent(datatable);

			insertsAndUpdatesAndExceptions.forEach(dvRow -> {
				dvRow.detach();
				// Add to response
				datatable.addContent(dvRow);
			});

		} catch (Throwable e) {
			if (log.isInfoEnabled() || e instanceof NullPointerException || e instanceof ArrayIndexOutOfBoundsException) {
				log.error("SaveData failed: {}", e.getMessage(), e); // Log NPEs!
			} else {
				log.error("SaveData failed: {}", e.getMessage(), e);
			}

			throw new RuntimeException("Failed to save data one-by-one", e);
		}
	}

	/**
	 * Declare extra CT columns for tracking row updates.
	 * @param magic target document
	 */
	private void expandDatatableSchemaForCT(final Element magic) {
		final Element dataviewSchemaRoot = xpDataviewSchema.evaluateFirst(magic);
		if (dataviewSchemaRoot != null) {
			Datatable.addRowMetadataSchemaElements(dataviewSchemaRoot);
		} else {
			log.warn("Could not find {}", xpDataviewSchema);
		}
	}

}