VirtualLookupServiceImpl.java

/*
 * Copyright 2022 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.gringlobal.compatibility.service.impl;

import java.lang.reflect.Field;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

import javax.annotation.Resource;
import javax.persistence.Id;

import com.querydsl.core.types.dsl.Expressions;
import com.querydsl.jpa.impl.JPAQuery;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.model.AuditedVersionedModel;
import org.genesys.blocks.security.model.AclSid;
import org.gringlobal.api.exception.InvalidApiUsageException;
import org.gringlobal.compatibility.LookupDisplay;
import org.gringlobal.compatibility.LookupValue;
import org.gringlobal.compatibility.SysTableInfo;
import org.gringlobal.compatibility.component.SysTableComponent;
import org.gringlobal.compatibility.service.VirtualLookupService;
import org.gringlobal.compatibility.service.impl.LookupServiceImpl.LookupStats;
import org.gringlobal.model.AuditedModel;
import org.gringlobal.soap.Datatable;
import org.gringlobal.soap.Datatable.HasChanges;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.convert.ConversionService;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.ReflectionUtils;

import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.BooleanExpression;
import com.querydsl.core.types.dsl.DatePath;
import com.querydsl.core.types.dsl.NumberPath;
import com.querydsl.core.types.dsl.PathBuilder;
import com.querydsl.jpa.impl.JPAQueryFactory;

/**
 * The Class VirtualLookupsServiceImpl.
 */
@Service
@Slf4j
public class VirtualLookupServiceImpl implements VirtualLookupService, InitializingBean {

	/** The Constant LOOKUP_SUFFIX. */
	public static final String LOOKUP_SUFFIX = "_lookup";

	/** The jpa query factory. */
	@Autowired
	protected JPAQueryFactory jpaQueryFactory;

	/** The entity class set. */
	@Resource
	private Set<Class<?>> entityClassSet;

	/** The conversion service. */
	@Autowired
	private ConversionService conversionService;


	/** The lookup value field cache. */
	private Map<Class<?>, Field> lookupValueFieldCache = new HashMap<>();
	/** The lookup display field cache. */
	private Map<Class<?>, Field> lookupDisplayFieldCache = new HashMap<>();

	@Autowired
	private SysTableComponent sysTableComponent;

	private Set<String> virtualLookups;

	@Override
	public void afterPropertiesSet() throws Exception {
		// Find all ggce_*_lookups
		var ggceLookups = new HashSet<String>();
		entityClassSet.stream().forEach(entityClass -> {
			if (! StringUtils.equals(entityClass.getPackageName(), "org.gringlobal.model")) {
				// Ignore non-GG models
				return;
			}
			var sysTableInfo = entityClass.getAnnotation(SysTableInfo.class);
			if (sysTableInfo != null && sysTableInfo.ignore()) {
				log.debug("Entity {} has @SysTableInfo(ignore = true)", entityClass.getSimpleName());
				return;
			}
			log.info("Scanning {} for virtual lookups", entityClass);
			var sysTableFields = sysTableComponent.getSysTableFields(entityClass);
			log.debug("\tGot {} fields", sysTableFields.size());
			sysTableFields.stream().filter(sysTableField -> sysTableField != null && StringUtils.isNotBlank(sysTableField.getForeignKeyDataviewName())).forEach(fieldWithLookup -> {
				log.info("\tRegistering {} in {}#{}", fieldWithLookup.getForeignKeyDataviewName(), entityClass, fieldWithLookup.getFieldName());
				ggceLookups.add(fieldWithLookup.getForeignKeyDataviewName());
			});
		});
		log.info("Virtual dataviews: {}", ggceLookups);
		this.virtualLookups = Collections.unmodifiableSet(ggceLookups);
	}

	@Override
	@PreAuthorize("isAuthenticated()")
	public void addAllLookupTableStats(Datatable result) {
		var mappedLookups = result.getRows().stream().map(row -> row.getValue(4)).collect(Collectors.toUnmodifiableSet());
		log.debug("Already mapped sys_table lookups {}", mappedLookups);

		for (var virtualLookup : virtualLookups) {
			final String dataviewName = virtualLookup;
			log.debug("Considering lookup {}", dataviewName);

			String entityName = StringUtils.removeStartIgnoreCase(StringUtils.removeEndIgnoreCase(dataviewName, LOOKUP_SUFFIX), VirtualDataviewServiceImpl.GGCE_PREFIX);

			Optional<Class<?>> optionalEntityClass = entityClassSet.stream().filter(entity -> StringUtils.equalsIgnoreCase(entity.getSimpleName(), entityName)).findFirst();
			if (! optionalEntityClass.isPresent()) {
				log.info("Dataview entity not found for {}", entityName);
				continue;
			}

			Class<?> entityClass = optionalEntityClass.get();

			var sysTable = sysTableComponent.getSysTable(entityClass);
			if (sysTable == null) {
				log.debug("{} is excluded from SOAP", entityClass);
				continue;
			}

//			if (mappedLookups.contains(sysTable.getTableName())) {
//				LOG.info("Lookup for table {} is already provided, skipping {}", sysTable.getTableName(), dataviewName);
//				continue;
//			}

			var sysTablePkField = sysTableComponent.getSysTableField(ReflectionUtils.findField(entityClass, "id"), entityClass);
			final LookupStats tableStats = getLookupTableStats(entityClass);

			result.addRow(
				HasChanges.original,
				//
				dataviewName,
				//
				entityName + " Lookup*", // dataview.get("title"),
				//
				"GGCE Virtual lookup for " + entityName, // dataview.get("description"),
				//
				sysTablePkField.getFieldName(), // dataview.get("pk_field_name"),
				//
				sysTable.getTableName(), // tableName,
				//
				tableStats.minPk,
				//
				tableStats.maxPk,
				//
				tableStats.rowCount,
				//
				tableStats.maxModifiedDate,
				//
				tableStats.maxCreatedDate,
				//
				tableStats.lastTouchedDate
			);
		}

	}

	private LookupStats getLookupTableStats(Class<?> entityClass) {
		log.debug("getLookupTableStats for {}", entityClass);

		PathBuilder<Object> root = new PathBuilder<>(entityClass, "t");
		var q = jpaQueryFactory.selectFrom(root);

		var pkPath = root.getNumber("id", Long.class);
		var createdDatePath = root.getDateTime("createdDate", Instant.class).max();
		var modifiedDatePath = root.getDateTime("modifiedDate", Instant.class).max();
		if (AuditedVersionedModel.class.isAssignableFrom(entityClass)) {
			modifiedDatePath = root.getDateTime("lastModifiedDate", Instant.class).max();
		}
		q.select(pkPath.min(), pkPath.max(), pkPath.count(), createdDatePath, modifiedDatePath, modifiedDatePath);

		Tuple stats = (Tuple) q.fetchOne();

		return new LookupStats(stats.get(0, Long.class), stats.get(1, Long.class), stats.get(2, Long.class),
				stats.get(createdDatePath), stats.get(modifiedDatePath), stats.get(modifiedDatePath));
	}

	/**
	 * Gets the lookup data.
	 *
	 * @param dataviewName the dataview name
	 * @param parameters the parameters
	 * @param offset the offset
	 * @param limit the limit
	 * @param options the options
	 * @return the lookup data
	 */
	@Override
	@Transactional(readOnly = true)
	@PreAuthorize("isAuthenticated()")
	public Datatable getLookupData(String dataviewName, Map<String, String> parameters, int offset, int limit, String options) {
		String entityName = StringUtils.removeEndIgnoreCase(StringUtils.removeStartIgnoreCase(dataviewName, VirtualDataviewServiceImpl.GGCE_PREFIX), LOOKUP_SUFFIX);

		Optional<Class<?>> optionalEntityClass = entityClassSet.stream().filter(entity -> StringUtils.equalsIgnoreCase(entity.getSimpleName(), entityName)).findFirst();
		if (! optionalEntityClass.isPresent()) {
			throw new InvalidApiUsageException("Dataview entity not found for name " + entityName);
		}

		Class<?> entityClass = optionalEntityClass.get();

		Field lookupValueField = getLookupValueField(entityClass);
		if (lookupValueField == null) {
			log.error("No lookupValueField for {}", entityClass);
			return null;
		}

		LookupFilter lookupFilter = new LookupFilter();
		lookupFilter.createdDate = getDateParameter(parameters.get(":createddate"));
		lookupFilter.modifiedDate = getDateParameter(parameters.get(":modifieddate"));
		String valueMemberParameter = parameters.get(":valuemember");
		if (StringUtils.isNotEmpty(valueMemberParameter)) {
			var parameterType = lookupValueField.getType();
			lookupFilter.valueMember = Arrays.stream(valueMemberParameter.split(","))
				.map(param -> conversionService.convert(param, parameterType)).filter(Objects::nonNull).collect(Collectors.toSet());
		}
		lookupFilter.startPKey = conversionService.convert(parameters.get(":startpkey"), Long.class);
		lookupFilter.stopPKey = conversionService.convert(parameters.get(":stoppkey"), Long.class);

		PathBuilder<Object> root = new PathBuilder<>(entityClass, "t");

		JPAQuery q;
		
		Field lookupDisplayField = getLookupDisplayField(entityClass);
		String lookupDisplayTemplate = null;

		if (AclSid.class.isAssignableFrom(entityClass)) { // Special case!
			lookupDisplayField = ReflectionUtils.findField(entityClass, "sid");
		}

		if (lookupDisplayField == null) {
			var lookupDisplayTypeAnnotation = entityClass.getAnnotation(LookupDisplay.class);
			if (lookupDisplayTypeAnnotation != null) {
				lookupDisplayTemplate = lookupDisplayTypeAnnotation.template();
			}
			if (lookupDisplayTemplate == null || StringUtils.isBlank(lookupDisplayTemplate)) {
				lookupDisplayField = lookupValueField; // Use id if @LookupDisplay.template is not defined
			}
		}

		if (lookupDisplayField != null) {
			var theLookupDisplay = root.get(lookupDisplayField.getName());
			if (entityClassSet.contains(lookupDisplayField.getType())) {
				log.warn("@LookupDisplay refers to an Entity: {}", lookupDisplayField.getType());
				Field entityLookupDisplayField = getLookupDisplayField(lookupDisplayField.getType());
				if (entityLookupDisplayField != null) {
					log.warn("@LookupDisplay uses {}.{}", entityLookupDisplayField.getDeclaringClass(), entityLookupDisplayField.getName());
					theLookupDisplay = theLookupDisplay.get(entityLookupDisplayField.getName());
				} else {
					throw new RuntimeException("@LookupDisplay is not provided on a field of " + lookupDisplayField.getType());
				}
			}
			q = jpaQueryFactory.select(root.get(lookupValueField.getName()), theLookupDisplay).from(root);
		} else {
			q = jpaQueryFactory.select(root.get(lookupValueField.getName()), Expressions.stringTemplate(lookupDisplayTemplate)).from(root);
		}

		BooleanExpression predicate = lookupFilter.buildPredicate(root, lookupValueField);
		if (Objects.nonNull(predicate)) {
			q = (JPAQuery) q.where(predicate);
		}

		// Apply limits
		if (limit > 0) {
			q.limit(limit);
		}
		if (offset > 1) {
			q.offset(offset);
		}

		List<Object> fetchedData = q.fetch();
		log.trace("Got {} matching lookup objects", fetchedData.size());
		Datatable datatable = new Datatable(dataviewName, List.of(
			new Datatable.Column("value_member", lookupValueField.getType()),
			new Datatable.Column("display_member", String.class)
		));
		datatable.setReadonly("Y");
		var valueMemberCol = datatable.getColumn("value_member");
		valueMemberCol.setProp("is_primary_key", "Y"); // TODO Maybe use sysTableField?

		if (fetchedData.isEmpty()) {
			return datatable;
		}

		for (var data: fetchedData) {
			Object valueMember = ((Tuple)data).get(0, lookupValueField.getType());
			Object displayMember;

			if (lookupDisplayTemplate != null) {
				displayMember = ((Tuple)data).get(1, String.class);
			} else {
				displayMember = ((Tuple)data).get(1, lookupDisplayField.getType());
			}
			datatable.addRow(HasChanges.original, valueMember, Objects.toString(displayMember == null ? valueMember : displayMember)); // Send valueMember if displayMember is null
		}
		log.debug("Generated {} rows for {}", datatable.getRows().size(), datatable.getName());
//		datatable.acceptChanges(); // All rows are original
		return datatable;
	}

	/**
	 * Gets the lookup display field.
	 *
	 * @param entityClass the entity class
	 * @return the lookup display field
	 */
	private Field getLookupDisplayField(Class<?> entityClass) {
		return lookupDisplayFieldCache.computeIfAbsent(entityClass, entity -> {
			var lookup = new AtomicReference<Field>();
			// Find lookupDisplay field
			ReflectionUtils.doWithFields(entity, field -> {
				if (field.isAnnotationPresent(LookupDisplay.class)) {
					ReflectionUtils.makeAccessible(field);
					lookup.set(field);
				}
			});
			return lookup.get();
		});
	}

	/**
	 * Gets the lookup value field.
	 *
	 * @param entityClass the entity class
	 * @return the lookup value field
	 */
	private Field getLookupValueField(Class<?> entityClass) {
		return lookupValueFieldCache.computeIfAbsent(entityClass, entity -> {
			var lookup = new AtomicReference<Field>();
			// Find lookupValue field
			ReflectionUtils.doWithFields(entity, field -> {
				if (field.isAnnotationPresent(LookupValue.class)) {
					ReflectionUtils.makeAccessible(field);
					lookup.set(field);
				}
			});
			// If @LookupValue is not present, use @Id
			if (lookup.get() == null) {
				ReflectionUtils.doWithFields(entity, field -> {
					if (field.isAnnotationPresent(Id.class)) {
						ReflectionUtils.makeAccessible(field);
						lookup.set(field);
					}
				});
			}
			return lookup.get();
		});
	}

	/**
	 * Gets the date parameter.
	 *
	 * @param dateParameter the date parameter
	 * @return the date parameter
	 */
	private Instant getDateParameter(String dateParameter) {
		Instant date = null;
		if (StringUtils.isNotEmpty(dateParameter)) {
			try {
				var dt = DateTimeFormatter.ISO_DATE_TIME.parseBest(dateParameter, OffsetDateTime::from, ZonedDateTime::from, LocalDateTime::from, LocalDate::from);
				if (dt instanceof ZonedDateTime) {
					return ((ZonedDateTime) dt).toInstant();
				} else if (dt instanceof OffsetDateTime) {
					return ((OffsetDateTime) dt).toInstant();
				} else if (dt instanceof LocalDateTime) {
					return ((LocalDateTime) dt).toInstant(ZoneOffset.UTC);
				} else if (dt instanceof LocalDate) {
					return ((LocalDate) dt).atStartOfDay().toInstant(ZoneOffset.UTC);
				} else {
					throw new DateTimeParseException("IDK", dateParameter, 0);
				}
			} catch (DateTimeParseException e) {
				log.warn("Exception in parsing given date parameter={}: {}", dateParameter, e.getMessage());
				throw e;
			}
		}
		return date;
	}

	/**
	 * Lookup parameters.
	 */
	static class LookupFilter {

		/** Minimum createdDate. */
		public Instant createdDate;

		/** Minimum modifiedDate. */
		public Instant modifiedDate;

		/**
		 * Set of valueMember to search for.
		 */
		public Set<Object> valueMember;

		/** Minimum PK inclusive. */
		public Long startPKey;

		/** Maximum PK exclusive. */
		public Long stopPKey;

//	/**
//	 * Set of displayMember
//	 */
//	public Set<String> displayMember; // Appears not to be used in any GG *_lookup dataview SQL queries

		/**
		 * Builds the predicate.
		 *
		 * @param root the root
		 * @param lookupValueField the lookup value field
		 * @return the boolean expression
		 */
		@SuppressWarnings("unchecked")
		public BooleanExpression buildPredicate(PathBuilder<Object> root, Field lookupValueField) {
			BooleanExpression predicate = null;

			if (Objects.nonNull(createdDate)) {
				DatePath<Instant> createdDatePath = root.getDate("createdDate", Instant.class);
				predicate = addPredicate(predicate, createdDatePath.gt(createdDate));
			}

			if (Objects.nonNull(modifiedDate)) {
				DatePath<Instant> modifiedDatePath = null;
				if (AuditedModel.class.isAssignableFrom(root.getType())) {
					modifiedDatePath = root.getDate("modifiedDate", Instant.class);
					predicate = addPredicate(predicate, modifiedDatePath.gt(modifiedDate));
				} else if (AuditedVersionedModel.class.isAssignableFrom(root.getType())) {
					modifiedDatePath = root.getDate("lastModifiedDate", Instant.class);
					predicate = addPredicate(predicate, modifiedDatePath.gt(modifiedDate));
				} else {
					log.warn("Missing modifiedDate on " + root.getType());
				}
			}

			if (Objects.nonNull(valueMember) && !valueMember.isEmpty()) {
				@SuppressWarnings("rawtypes")
				PathBuilder valueMemberPath = root.get(lookupValueField.getName(), lookupValueField.getType());
				predicate = addPredicate(predicate, valueMemberPath.in(valueMember));
			}

			if (Objects.nonNull(startPKey) && Objects.nonNull(stopPKey)) {
				NumberPath<Long> pkPath = root.getNumber("id", Long.class);
				predicate = addPredicate(predicate, pkPath.between(startPKey, stopPKey));
			}

			return predicate;
		}

		/**
		 * Adds the predicate.
		 *
		 * @param target the target
		 * @param predicate the predicate
		 * @return the boolean expression
		 */
		private BooleanExpression addPredicate(BooleanExpression target, BooleanExpression predicate) {
			if (target != null) {
				log.trace("OR {}", predicate);
				return target.or(predicate);
			}
			log.trace("{}", predicate);
			return predicate;
		}

	}

}