DataviewServicesImpl.java

/*
 * Copyright 2021 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.gringlobal.service.impl;

import java.io.IOException;
import java.io.InputStream;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.common.util.set.Sets;
import org.gringlobal.api.exception.NotFoundElement;
import org.gringlobal.model.QSysDataviewField;
import org.gringlobal.model.QSysDataviewFieldLang;
import org.gringlobal.model.QSysDataviewLang;
import org.gringlobal.model.QSysDataviewParam;
import org.gringlobal.model.QSysDataviewSql;
import org.gringlobal.model.SysDataview;
import org.gringlobal.model.SysDataviewField;
import org.gringlobal.model.SysDataviewFieldLang;
import org.gringlobal.model.SysDataviewLang;
import org.gringlobal.model.SysDataviewParam;
import org.gringlobal.model.SysDataviewSql;
import org.gringlobal.model.SysLang;
import org.gringlobal.persistence.SysDataviewFieldLangRepository;
import org.gringlobal.persistence.SysDataviewFieldRepository;
import org.gringlobal.persistence.SysDataviewLangRepository;
import org.gringlobal.persistence.SysDataviewParamRepository;
import org.gringlobal.persistence.SysDataviewRepository;
import org.gringlobal.persistence.SysDataviewSqlRepository;
import org.gringlobal.persistence.SysTableFieldRepository;
import org.gringlobal.service.DataviewServices;
import org.gringlobal.service.LanguageService;
import org.gringlobal.service.SysTableMappingException;
import org.gringlobal.service.TableServices.SysTableFieldService;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.JDOMException;
import org.jdom2.input.SAXBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

/**
 * The Class DataviewServicesImpl.
 */
@Service
@Slf4j
public class DataviewServicesImpl implements DataviewServices {

	/**
	 * The Class SysDataviewServiceImpl.
	 */
	@Service
	@Transactional(readOnly = true)
	public static class SysDataviewServiceImpl extends CRUDServiceImpl<SysDataview, SysDataviewRepository> implements SysDataviewService {

		@Autowired
		private SysDataviewFieldRepository dataviewFieldRepository;

		@Autowired
		private LanguageService languageService;

		@Autowired
		private SysDataviewLangService sysDataviewLangService;

		@Autowired
		private SysDataviewSqlRepository sysDataviewSqlRepository;

		@Autowired
		private SysDataviewLangRepository dataviewLangRepository;

		@Autowired
		private SysDataviewParamRepository dataviewParamRepository;

		@Autowired
		private SysTableFieldRepository sysTableFieldRepository;

		@Autowired
		private SysDataviewFieldLangRepository fieldLangRepository;

		@Autowired
		private SysTableFieldService sysTableFieldService;

		@Override
		public SysDataview load(String dataviewName) {
			return _lazyLoad(repository.findByDataviewName(dataviewName));
		}

		@Override
		@Transactional
		public SysDataview create(SysDataview source) {
			return _lazyLoad(repository.save(source));
		}

		@Override
		@Transactional
		public SysDataview create(SysDataview source, String title, Map<String, String> sqlStatements) {
			var sysDataview = create(source);
			SysDataviewLang lang = new SysDataviewLang();
			lang.setSysLang(new SysLang(1l));
			lang.setTitle(title);
			lang.setEntity(sysDataview);
			sysDataviewLangService.create(lang);

			sqlStatements.forEach((sqlEngine, sqlStatement) -> {
				SysDataviewSql sql = new SysDataviewSql();
				sql.setDatabaseEngineTag(sqlEngine);
				sql.setSqlStatement(sqlStatement);
				sql.setDataview(sysDataview);
				sysDataviewSqlRepository.save(sql);
			});

			return reload(sysDataview);
		}

		@Override
		@Transactional
		public SysDataview update(SysDataview updated, SysDataview target) {
			target.apply(updated);
			return _lazyLoad(repository.save(target));
		}

		@Override
		@Transactional
		public SysDataview updateDataviewFromXML(InputStream inputStream, SysDataview target) throws JDOMException, IOException {
			target = reload(target);

			SAXBuilder saxBuilder = new SAXBuilder();
			Document document = saxBuilder.build(inputStream);
			Element rootElement = document.getRootElement();
			Element sysDataviewEl = rootElement.getChild("sys_dataview");

			target.setDataviewName(xmlTextOf(sysDataviewEl, "dataview_name"));
			target.setIsEnabled(xmlTextOf(sysDataviewEl, "is_enabled"));
			target.setIsReadonly(xmlTextOf(sysDataviewEl, "is_readonly"));
			target.setIsTransform(xmlTextOf(sysDataviewEl, "is_transform"));
			target.setCategoryCode(xmlTextOf(sysDataviewEl, "category_code"));
			target.setDatabaseAreaCode(xmlTextOf(sysDataviewEl, "database_area_code"));
			target.setTransformFieldForNames(xmlTextOf(sysDataviewEl, "transform_field_for_names"));
			target.setTransformFieldForCaptions(xmlTextOf(sysDataviewEl, "transform_field_for_captions"));
			target.setTransformFieldForValues(xmlTextOf(sysDataviewEl, "transform_field_for_values"));
			target.setConfigurationOptions(xmlTextOf(sysDataviewEl, "configuration_options"));

			String databaseAreaCodeSortOrderStr = xmlTextOf(sysDataviewEl, "database_area_code_sort_order");
			target.setDatabaseAreaCodeSortOrder(databaseAreaCodeSortOrderStr != null ? Integer.parseInt(databaseAreaCodeSortOrderStr) : null);

			SysDataview updatedSysDataview = repository.save(target);

			HashSet<SysDataviewLang> currentLangs = Sets.newHashSet(dataviewLangRepository.findAll(QSysDataviewLang.sysDataviewLang.entity().eq(updatedSysDataview)));
			rootElement.getChildren("sys_dataview_lang").stream().map(e -> upsertSysDataviewLang(e, updatedSysDataview)).forEach(currentLangs::remove);
			dataviewLangRepository.deleteAllInBatch(currentLangs);

			HashSet<SysDataviewSql> currentSQLs = Sets.newHashSet(sysDataviewSqlRepository.findAll(QSysDataviewSql.sysDataviewSql.dataview().eq(updatedSysDataview)));
			rootElement.getChildren("sys_dataview_sql").stream().map(e -> upsertSysDataviewSql(e, updatedSysDataview)).forEach(currentSQLs::remove);
			sysDataviewSqlRepository.deleteAllInBatch(currentSQLs);

			HashSet<SysDataviewParam> currentParams = Sets.newHashSet(dataviewParamRepository.findAll(QSysDataviewParam.sysDataviewParam.dataview().eq(updatedSysDataview)));
			AtomicInteger sortOrder = new AtomicInteger(0);
			rootElement.getChildren("sys_dataview_param").stream()
				// convert
				.map(e -> upsertSysDataviewParam(e, updatedSysDataview))
				// sort
				.sorted(Comparator.comparingInt(SysDataviewParam::getSortOrder))
				// store
				.forEach(sysDataviewParam -> {
					sysDataviewParam.setSortOrder(sortOrder.getAndIncrement());
					currentParams.remove(dataviewParamRepository.save(sysDataviewParam));
				});
			dataviewParamRepository.deleteAllInBatch(currentParams);
			sortOrder.set(0); // reset

			HashSet<SysDataviewField> currentFields = Sets.newHashSet(dataviewFieldRepository.findAll(QSysDataviewField.sysDataviewField.dataview().eq(updatedSysDataview)));
			List<Element> sysDataviewFieldLangs = rootElement.getChildren("sys_dataview_field_lang");
			rootElement.getChildren("sys_dataview_field").stream()
				// convert
				.map(e -> upsertSysDataviewField(e, updatedSysDataview))
				// sort
				.sorted(Comparator.comparingInt(SysDataviewField::getSortOrder))
				// store
				.forEach(sysDataviewField -> {
					sysDataviewField.setSortOrder(sortOrder.getAndIncrement());
					var savedSysDataviewField = dataviewFieldRepository.save(sysDataviewField);
					currentFields.remove(savedSysDataviewField);

					HashSet<SysDataviewFieldLang> currentFieldLangs = Sets.newHashSet(fieldLangRepository.findAll(QSysDataviewFieldLang.sysDataviewFieldLang.entity().eq(sysDataviewField)));

					sysDataviewFieldLangs.stream().filter(el -> Objects.equals(el.getChild("field_name").getValue(), savedSysDataviewField
						.getFieldName())).forEach(el -> {
						SysDataviewFieldLang sysDataviewFieldLang = upsertSysDataviewFieldLang(el, savedSysDataviewField);
						currentFieldLangs.remove(sysDataviewFieldLang);
					});

					fieldLangRepository.deleteAllInBatch(currentFieldLangs);
				});

			dataviewFieldRepository.deleteAll(currentFields);

			return updatedSysDataview;
		}

		@Override
		@Transactional(noRollbackFor = { NotFoundElement.class })
		public SysDataview registerDataviewFromXML(InputStream inputStream) throws JDOMException, IOException {
			SAXBuilder saxBuilder = new SAXBuilder();
			Document document = saxBuilder.build(inputStream);
			Element rootElement = document.getRootElement();

			Element sysDataviewEl = rootElement.getChild("sys_dataview");
			SysDataview sysDataview = new SysDataview();
			sysDataview.setDataviewName(xmlTextOf(sysDataviewEl, "dataview_name"));
			sysDataview.setIsEnabled(xmlTextOf(sysDataviewEl, "is_enabled"));
			sysDataview.setIsReadonly(xmlTextOf(sysDataviewEl, "is_readonly"));
			sysDataview.setIsTransform(xmlTextOf(sysDataviewEl, "is_transform"));
			sysDataview.setCategoryCode(xmlTextOf(sysDataviewEl, "category_code"));
			sysDataview.setDatabaseAreaCode(xmlTextOf(sysDataviewEl, "database_area_code"));
			sysDataview.setTransformFieldForNames(xmlTextOf(sysDataviewEl, "transform_field_for_names"));
			sysDataview.setTransformFieldForCaptions(xmlTextOf(sysDataviewEl, "transform_field_for_captions"));
			sysDataview.setTransformFieldForValues(xmlTextOf(sysDataviewEl, "transform_field_for_values"));
			sysDataview.setConfigurationOptions(xmlTextOf(sysDataviewEl, "configuration_options"));

			String databaseAreaCodeSortOrderStr = xmlTextOf(sysDataviewEl, "database_area_code_sort_order");
			sysDataview.setDatabaseAreaCodeSortOrder(databaseAreaCodeSortOrderStr != null ? Integer.parseInt(databaseAreaCodeSortOrderStr) : null);

			SysDataview savedSysDataview = repository.save(sysDataview);

			rootElement.getChildren("sys_dataview_lang").forEach(e -> registerSysDataviewLang(e, savedSysDataview));
			rootElement.getChildren("sys_dataview_sql").forEach(e -> registerSysDataviewSql(e, savedSysDataview));

			AtomicInteger sortOrder = new AtomicInteger(0);
			rootElement.getChildren("sys_dataview_param").stream()
				// convert
				.map(e -> registerSysDataviewParam(e, savedSysDataview))
				// sort
				.sorted((a, b) -> Integer.compare(a.getSortOrder(), b.getSortOrder()))
				// store
				.forEach(sysDataviewParam -> {
					sysDataviewParam.setSortOrder(sortOrder.getAndIncrement());
					dataviewParamRepository.save(sysDataviewParam);
				});

			sortOrder.set(0); // reset
			List<Element> sysDataviewFieldLangs = rootElement.getChildren("sys_dataview_field_lang");
			rootElement.getChildren("sys_dataview_field").stream()
				// convert
				.map(e -> registerSysDataviewField(e, savedSysDataview))
				// sort
				.sorted((a, b) -> Integer.compare(a.getSortOrder(), b.getSortOrder()))
				// store
				.forEach(sysDataviewField -> {
					sysDataviewField.setSortOrder(sortOrder.getAndIncrement());
					dataviewFieldRepository.save(sysDataviewField);
					List<SysDataviewFieldLang> langs = sysDataviewFieldLangs.stream().filter(el -> Objects.equals(el.getChild("field_name").getValue(), sysDataviewField
							.getFieldName())).map(el -> registerSysDataviewFieldLang(el, sysDataviewField))
						// keep not-nulls
						.filter(lang -> lang != null).collect(Collectors.toList());
					sysDataviewField.setLangs(langs);
				});

			return savedSysDataview;
		}

		private String xmlTextOf(Element xmlNode, String nodeName) {
			var node = xmlNode.getChild(nodeName);
			return node != null ? StringUtils.trimToNull(node.getValue()) : null;
		}

		private SysDataviewLang upsertSysDataviewLang(Element element, SysDataview dataview) {
			SysLang sysLang = languageService.getLanguage(element.getChild("ietf_tag").getValue());
			var existing = dataviewLangRepository.getByEntityAndSysLang(dataview, sysLang);
			if (existing != null) {
				existing.setTitle(xmlTextOf(element, "title"));
				existing.setDescription(xmlTextOf(element, "description"));
				return dataviewLangRepository.save(existing);
			} else return registerSysDataviewLang(element, dataview);
		}

		private SysDataviewLang registerSysDataviewLang(Element element, SysDataview dataview) {
			SysLang sysLang = languageService.getLanguage(element.getChild("ietf_tag").getValue());
			if (sysLang == null) {
				log.warn("Could not register SysDataviewLang, no language {}", element.getChild("ietf_tag").getValue());
				return null;
			}
			SysDataviewLang sysDataviewLang = new SysDataviewLang();
			sysDataviewLang.setEntity(dataview);

			sysDataviewLang.setTitle(xmlTextOf(element, "title"));
			sysDataviewLang.setDescription(xmlTextOf(element, "description"));

			sysDataviewLang.setSysLang(sysLang);
			return dataviewLangRepository.save(sysDataviewLang);
		}

		private SysDataviewField upsertSysDataviewField(Element element, SysDataview dataview) {
			String fieldName = xmlTextOf(element, "field_name");
			SysDataviewField existing = dataviewFieldRepository.findByDataviewAndFieldName(dataview, fieldName);
			if (existing != null) {
				existing.setFieldName(xmlTextOf(element, "field_name"));
				existing.setIsReadonly(xmlTextOf(element, "is_readonly"));
				existing.setIsPrimaryKey(xmlTextOf(element, "is_primary_key"));
				existing.setIsTransform(xmlTextOf(element, "is_transform"));
				existing.setTableAliasName(xmlTextOf(element, "table_alias_name"));
				existing.setIsVisible(xmlTextOf(element, "is_visible"));
				existing.setGuiHint(xmlTextOf(element, "gui_hint"));
				existing.setForeignKeyDataviewName(xmlTextOf(element, "foreign_key_dataview_name"));
				existing.setConfigurationOptions(xmlTextOf(element, "configuration_options"));
				existing.setGroupName(xmlTextOf(element, "group_name"));
				existing.setSortOrder(Integer.parseInt(xmlTextOf(element, "sort_order")));

				String tableName = xmlTextOf(element, "table_name");
				String tableFieldName = xmlTextOf(element, "table_field_name");
				if (StringUtils.isNotBlank(tableName) && StringUtils.isNotBlank(tableFieldName)) {
					var sysTableField = sysTableFieldRepository.findByTableNameAndFieldName(tableName, tableFieldName);
					if (sysTableField == null) {
						log.warn("Unmapped SysTableField {}.{}", tableName, tableFieldName);
						try {
							sysTableField = sysTableFieldService.generateMapping(tableName, tableFieldName);
						} catch (SysTableMappingException e) {
							log.error("Could not generate mapping for {}.{}: {}", tableName, tableFieldName, e.getMessage());
						}
					}
					existing.setSysTableField(sysTableField);
				}
				return existing;
			} else {
				log.warn("Registering a new SysDataviewField {}.{}", dataview, fieldName);
				return registerSysDataviewField(element, dataview);
			}
		}

		private SysDataviewField registerSysDataviewField(Element element, SysDataview dataview) {
			SysDataviewField sysDataviewField = new SysDataviewField();
			sysDataviewField.setDataview(dataview);

			sysDataviewField.setFieldName(xmlTextOf(element, "field_name"));
			sysDataviewField.setIsReadonly(xmlTextOf(element, "is_readonly"));
			sysDataviewField.setIsPrimaryKey(xmlTextOf(element, "is_primary_key"));
			sysDataviewField.setIsTransform(xmlTextOf(element, "is_transform"));
			sysDataviewField.setTableAliasName(xmlTextOf(element, "table_alias_name"));
			sysDataviewField.setIsVisible(xmlTextOf(element, "is_visible"));
			sysDataviewField.setGuiHint(xmlTextOf(element, "gui_hint"));
			sysDataviewField.setForeignKeyDataviewName(xmlTextOf(element, "foreign_key_dataview_name"));
			sysDataviewField.setConfigurationOptions(xmlTextOf(element, "configuration_options"));
			sysDataviewField.setGroupName(xmlTextOf(element, "group_name"));
			sysDataviewField.setSortOrder(Integer.parseInt(xmlTextOf(element, "sort_order")));

			String tableName = xmlTextOf(element, "table_name");
			String tableFieldName = xmlTextOf(element, "table_field_name");
			if (StringUtils.isNotBlank(tableName) && StringUtils.isNotBlank(tableFieldName)) {
				var sysTableField = sysTableFieldRepository.findByTableNameAndFieldName(tableName, tableFieldName);
				if (sysTableField == null) {
					log.warn("Unmapped SysTableField {}.{}", tableName, tableFieldName);
					try {
						sysTableField = sysTableFieldService.generateMapping(tableName, tableFieldName);
					} catch (SysTableMappingException e) {
						log.error("Could not generate mapping for {}.{}: {}", tableName, tableFieldName, e.getMessage());
					}
				}
				sysDataviewField.setSysTableField(sysTableField);
			}

			return sysDataviewField;
		}

		private SysDataviewFieldLang upsertSysDataviewFieldLang(Element element, SysDataviewField entity) {
			SysLang sysLang = languageService.getLanguage(element.getChild("ietf_tag").getValue());
			var existing = fieldLangRepository.getByEntityAndSysLang(entity, sysLang);
			if (existing != null) {
				existing.setTitle(xmlTextOf(element, "title"));
				existing.setDescription(xmlTextOf(element, "description"));
				return fieldLangRepository.save(existing);
			} else return registerSysDataviewFieldLang(element, entity);
		}

		private SysDataviewFieldLang registerSysDataviewFieldLang(Element element, SysDataviewField entity) {
			SysLang sysLang = languageService.getLanguage(element.getChild("ietf_tag").getValue());
			if (sysLang == null) {
				log.warn("Will not register SysDataviewFieldLang, missing language {}", element.getChild("ietf_tag").getValue());
				return null;
			}

			SysDataviewFieldLang lang = new SysDataviewFieldLang();
			lang.setSysLang(sysLang);
			lang.setEntity(entity);
			lang.setTitle(xmlTextOf(element, "title"));
			lang.setDescription(xmlTextOf(element, "description"));

			return fieldLangRepository.save(lang);
		}

		private SysDataviewParam upsertSysDataviewParam(Element element, SysDataview dataview) {
			String paramName = xmlTextOf(element, "param_name");

			var existing = dataviewParamRepository.findByDataviewAndParamName(dataview, paramName);
			if (existing != null) {
				existing.setParamType(xmlTextOf(element, "param_type"));

				String sortOrder = xmlTextOf(element, "sort_order");
				existing.setSortOrder(sortOrder != null ? Integer.parseInt(sortOrder) : null);

				return existing;
			} else return registerSysDataviewParam(element, dataview);
		}

		private SysDataviewParam registerSysDataviewParam(Element element, SysDataview dataview) {
			SysDataviewParam sysDataviewParam = new SysDataviewParam();
			sysDataviewParam.setDataview(dataview);

			sysDataviewParam.setParamName(xmlTextOf(element, "param_name"));
			sysDataviewParam.setParamType(xmlTextOf(element, "param_type"));

			String sort_order = xmlTextOf(element, "sort_order");
			sysDataviewParam.setSortOrder(sort_order != null ? Integer.parseInt(sort_order) : null);

			return sysDataviewParam;
		}

		private SysDataviewSql upsertSysDataviewSql(Element element, SysDataview dataview) {
			String databaseEngineTag = xmlTextOf(element, "database_engine_tag");
			SysDataviewSql existing = sysDataviewSqlRepository.findByDataviewAndDatabaseEngineTag(dataview, databaseEngineTag);
			if (existing != null) {
				existing.setSqlStatement(xmlTextOf(element, "sql_statement"));
				return sysDataviewSqlRepository.save(existing);
			} else return registerSysDataviewSql(element, dataview);
		}

		private SysDataviewSql registerSysDataviewSql(Element element, SysDataview dataview) {
			SysDataviewSql sysDataviewSql = new SysDataviewSql();
			sysDataviewSql.setDataview(dataview);

			sysDataviewSql.setDatabaseEngineTag(xmlTextOf(element, "database_engine_tag"));
			sysDataviewSql.setSqlStatement(xmlTextOf(element, "sql_statement"));
			return sysDataviewSqlRepository.save(sysDataviewSql);
		}

	}

	@Service
	@Transactional(readOnly = true)
	public static class SysDataviewLangServiceImpl extends CRUDServiceImpl<SysDataviewLang, SysDataviewLangRepository> implements SysDataviewLangService {

		@Override
		@Transactional
		public SysDataviewLang create(SysDataviewLang source) {
			return _lazyLoad(repository.save(source));
		}

		@Override
		@Transactional
		public SysDataviewLang update(SysDataviewLang updated, SysDataviewLang target) {
			return null;
		}
	}

}