GlisDOIRegistrationManager.java

/*
 * Copyright 2022 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.gringlobal.service.glis.impl;

import static org.gringlobal.service.glis.impl.GlisSMTAReportingManager.*;

import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.security.SecurityContextUtil;
import org.genesys.glis.v1.api.ManagerApi;
import org.genesys.glis.v1.model.Acquisition;
import org.genesys.glis.v1.model.Actor;
import org.genesys.glis.v1.model.BasePGRFA;
import org.genesys.glis.v1.model.Breeder;
import org.genesys.glis.v1.model.Breeding;
import org.genesys.glis.v1.model.Collection;
import org.genesys.glis.v1.model.Collector;
import org.genesys.glis.v1.model.Location;
import org.genesys.glis.v1.model.PGRFARegistration;
import org.genesys.glis.v1.model.PGRFARegistrationResponse;
import org.genesys.glis.v1.model.PGRFAUpdate;
import org.gringlobal.api.exception.InvalidApiUsageException;
import org.gringlobal.api.exception.NotFoundElement;
import org.gringlobal.model.community.AccessionMCPD;
import org.gringlobal.persistence.AccessionRepository;
import org.gringlobal.service.AccessionService;
import org.gringlobal.service.AppSettingsService;
import org.gringlobal.service.ShortFilterService;
import org.gringlobal.service.filter.AccessionFilter;
import org.gringlobal.spring.TransactionHelper;
import org.gringlobal.worker.AccessionMCPDConverter;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.client.HttpClientErrorException;

import com.fasterxml.jackson.dataformat.xml.XmlMapper;

import javax.validation.constraints.NotNull;

@Component
@Slf4j
public class GlisDOIRegistrationManager {

	private static final String APPSETTINGS_ITPGRFA_GLIS = "ITPGRFA_GLIS";

	private static final String ASSIGN_DOI_TO_MANY_ERRORS = "The DOI assignment operation was stopped due to multiple processing errors.";

	@Autowired
	private AccessionService accessionService;

	@Autowired
	private AccessionRepository accessionRepository;

	@Autowired
	private AppSettingsService appSettingsService;

	@Autowired
	private AccessionMCPDConverter accessionMCPDConverter;

	@Autowired
	@Qualifier("glisDOIManagerApiFactory")
	private FactoryBean<ManagerApi> managerApiFactory;

	@Autowired
	private ThreadPoolTaskExecutor executor;

	@Autowired
	private ShortFilterService shortFilterService;

	private final AtomicReference<AssignDoiState> assignDoiState = new AtomicReference<>();

	public static class AssignDoiState {
		public String filterCode;
		public AssignDoiProgressData progress;
		public boolean canceled;

		public AssignDoiState(String filterCode, AssignDoiProgressData progress) {
			this.filterCode = filterCode;
			this.progress = progress;
		}

		public AssignDoiState updateProgress(int total, List<GlisDoiResponse> processed, AssignDoiProgressData.Status status, String error) {
			this.progress.status = status;
			this.progress.total = total;
			this.progress.processed = processed;
			this.progress.error = error;
			return this;
		}

		public AssignDoiState updateProgress(int total, AssignDoiProgressData.Status status) {
			this.progress.status = status;
			this.progress.total = total;
			return this;
		}

		public AssignDoiState updateProgress(GlisDoiResponse processedToAdd, AssignDoiProgressData.Status status) {
			this.progress.status = status;
			this.progress.processed.add(processedToAdd);
			return this;
		}

		public AssignDoiState updateProgress(AssignDoiProgressData.Status status, String error) {
			this.progress.status = status;
			this.progress.error = error;
			return this;
		}

		public AssignDoiState updateProgress(AssignDoiProgressData.Status status) {
			this.progress.status = status;
			return this;
		}

		public AssignDoiState cancel() {
			this.canceled = true;
			return this;
		}
	}

	@Data
	public static class AssignDoiProgressData {
		public enum Status {UPLOADING, ABORTED, DONE}

		private Status status;
		private long total;
		private List<GlisDoiResponse> processed;
		private String error;

		public AssignDoiProgressData(Status status, long total, List<GlisDoiResponse> processed) {
			this.status = status;
			this.total = total;
			this.processed = processed;
		}
	}

	public static class GlisDoiResponse {
		public long accessionId;
		public String accessionNumber;
		public String doi;
		public String error;

		public GlisDoiResponse(long accessionId, String accessionNumber, String doi) {
			this.accessionId = accessionId;
			this.accessionNumber = accessionNumber;
			this.doi = doi;
		}
	}

	public ResponseEntity<HttpStatus> cancelDoiAssignment(@NotNull String filterCode) {

		if (assignDoiState.get() != null && assignDoiState.get().filterCode.equals(filterCode)) {
			// cancel only active uploading
			if (assignDoiState.get().progress.status == AssignDoiProgressData.Status.UPLOADING) {
				var sid = SecurityContextUtil.getCurrentUser() != null ? SecurityContextUtil.getCurrentUser().getSid() : null;
				log.warn("DOI assignment was canceled by {}", sid);
				assignDoiState.set(assignDoiState.get().cancel());
			}
		} else {
			throw new InvalidApiUsageException("No active assigning is currently running for requested ids.");
		}
		return ResponseEntity.ok().build();
	}

	/**
	 * Update GLIS DOI Registration service to register or update PGRFA. Registration of a new PGRFA will
	 * result in generation of a new DOI.
	 *
	 * @param filterCode filter code
	 * @param filter AccessionFilter
	 * @return AssignDoiProgressData result
	 */
	@PreAuthorize("@ggceSec.actionAllowed('PassportData', 'ADMINISTRATION')")
	@Transactional(readOnly = true)
	public List<GlisDoiResponse> updateDoiRegistration(String filterCode, AccessionFilter filter) throws Exception {
		final var filterInfo = shortFilterService.processFilter(filterCode, filter, AccessionFilter.class);
		if (filterInfo.filter.toString().equals("{}"))
			throw new InvalidApiUsageException("Refusing to upload accessions by empty filter.");

		return updateGlisRegistration(filterInfo);
	}

	/**
	 * Update GLIS DOI Registration service to register or update PGRFA. Registration of a new PGRFA will
	 * result in generation of a new DOI.
	 *
	 * @param filterCode filter code
	 * @param filter AccessionFilter
	 * @return AssignDoiProgressData result
	 */
	@PreAuthorize("@ggceSec.actionAllowed('PassportData', 'ADMINISTRATION')")
	@Transactional(readOnly = true)
	public AssignDoiState updateDoiRegistrationWithProgress(String filterCode, AccessionFilter filter) throws Exception {
		final var filterInfo = shortFilterService.processFilter(filterCode, filter, AccessionFilter.class);
		if (filterInfo.filter.toString().equals("{}"))
			throw new InvalidApiUsageException("Refusing to upload accessions by empty filter.");

		return updateGlisRegistrationWithProgress(filterInfo);
	}

	private List<GlisDoiResponse> updateGlisRegistration(ShortFilterService.FilterInfo<AccessionFilter> filterInfo) {

		if (assignDoiState.get() != null && assignDoiState.get().progress.status == AssignDoiProgressData.Status.UPLOADING) {
			// only one active upload at a time!
			throw new InvalidApiUsageException("Another assign doi process is currently running.");
		}

		assignDoiState.set(new AssignDoiState(filterInfo.filterCode, new AssignDoiProgressData(AssignDoiProgressData.Status.UPLOADING, 0, new LinkedList<>())));
		try {
			return assignDOI(filterInfo.filter);
		} catch (Throwable e) {
			log.warn("Error while DOI assignment: {}", e.getMessage());
			throw new InvalidApiUsageException("Error while DOI assignment.", e);
		}
	}

	private AssignDoiState updateGlisRegistrationWithProgress(ShortFilterService.FilterInfo<AccessionFilter> filterInfo) {

		if (assignDoiState.get() != null && assignDoiState.get().filterCode.equals(filterInfo.filterCode)) {
			if (assignDoiState.get().progress.status != AssignDoiProgressData.Status.UPLOADING) {
				// return progress if it's DONE or ABORTED, then clear it
				return assignDoiState.getAndSet(null);
			} else {
				return assignDoiState.get(); // return progress, still uploading...
			}
		} else if (assignDoiState.get() != null && !assignDoiState.get().filterCode.equals(filterInfo.filterCode)) {
			if (assignDoiState.get().progress.status == AssignDoiProgressData.Status.UPLOADING) {
				// only one active upload at a time!
				throw new InvalidApiUsageException("Another assign doi process is currently running.");
			}
		}

		executor.execute(() -> {
			assignDoiState.set(new AssignDoiState(filterInfo.filterCode, new AssignDoiProgressData(AssignDoiProgressData.Status.UPLOADING, 0, new LinkedList<>())));
			try {
				assignDOI(filterInfo.filter);
			} catch (Throwable e) {
				log.warn("Error while DOI assignment: {}", e.getMessage());
				if (StringUtils.isEmpty(assignDoiState.get().progress.error)) {
					assignDoiState.get().updateProgress(0, new LinkedList<>(), AssignDoiProgressData.Status.ABORTED, e.getLocalizedMessage());
				}
			}
		});
		return new AssignDoiState(filterInfo.filterCode, new AssignDoiProgressData(AssignDoiProgressData.Status.UPLOADING, 0, new LinkedList<>()));
	}

	private List<GlisDoiResponse> assignDOI(AccessionFilter filter) throws Exception {

		var managerApi = createGlisManager();

		String glisUsername = appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_USERNAME).getValue();
		String glisPassword = appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_PASSWORD).getValue();
		String glisInstitutePid = appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_PID).getValue();
		String glisInstituteName = appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_INSTITUTE_NAME).getValue();
		String glisInstituteAddress = appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_INSTITUTE_ADDRESS).getValue();
		String glisInstituteCountry = appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_INSTITUTE_COUNTRY_CODE).getValue();

		XmlMapper xmlMapper = new XmlMapper(); // For reading HTTP 400 errors
		AtomicInteger errorCounter = new AtomicInteger(0);

		boolean completed = TransactionHelper.executeInTransaction(true, () -> {
				var accessions = accessionService.list(filter, Pageable.unpaged()).getContent();
				var total = accessions.size();

				// Update total
				assignDoiState.get().updateProgress(total, AssignDoiProgressData.Status.UPLOADING);

				for (var accession : accessions) {
					if (assignDoiState.get().canceled) {
						throw new InvalidApiUsageException("Assignment has been canceled by the user.");
					}

					var doiUpdate = new GlisDoiResponse(accession.getId(), accession.getAccessionNumber(), accession.getDoi());

					// Skip if not web visible
					if (!Objects.equals("Y", accession.getIsWebVisible())) {
						doiUpdate.error = "Accession is not visible to external users";
						assignDoiState.get().updateProgress(doiUpdate, AssignDoiProgressData.Status.UPLOADING);
						continue;
					}
					// Skip if acquisition date missing
					if (accession.getInitialReceivedDate() == null) {
						doiUpdate.error = "Accession initialReceiedDate is not declared";
						assignDoiState.get().updateProgress(doiUpdate, AssignDoiProgressData.Status.UPLOADING);
						continue;
					}

					var accessionMCPD = accessionMCPDConverter.convert(accession);

					BasePGRFA request;

					if (StringUtils.isNotBlank(accessionMCPD.puid)) {
						PGRFAUpdate update = new PGRFAUpdate();
						update.setSampledoi(accessionMCPD.puid); // Current doi
						request = update;
					} else {
						// Build new registration request
						PGRFARegistration registration = new PGRFARegistration();
						request = registration;
					}

					request.setUsername(glisUsername);
					request.setPassword(glisPassword);

					Location location = new Location();
					location.setWiews(accessionMCPD.instCode);
					// Set holding institute
					location.setPid(glisInstitutePid);
					location.setName(glisInstituteName);
					location.setAddress(glisInstituteAddress);
					location.setCountry(glisInstituteCountry);
					request.setLocation(location);

					request.setSampleid(accessionMCPD.acceNumb);

					if (StringUtils.isNotBlank(accessionMCPD.acqDate)) {
						request.setDate(convertMcpdDateToRequest(accessionMCPD.acqDate));
					}
					// Declare method
					request.setMethod(determineMethod(accessionMCPD));
					request.setGenus(accessionMCPD.genus);
					request.setSpecies(accessionMCPD.species);
					if (StringUtils.isNotBlank(accessionMCPD.cropName)) {
						request.setCropnames(List.of(accessionMCPD.cropName));
					}

					if (StringUtils.isNotBlank(accessionMCPD.acceUrl)) {
						// Skip accession URL
//					Target target = new Target();
//					target.setValue(accessionMCPD.acceUrl);
//					// Define kws: 1-Passport data
//					target.setKws(List.of("1"));
//					request.setTargets(List.of(target));
					}

					request.setBiostatus(accessionMCPD.sampStat);
					request.setSpauth(accessionMCPD.spAuthor);
					request.setSubtaxa(accessionMCPD.subtaxa);
					request.setStauth(accessionMCPD.subtAuthor);

					if (StringUtils.isNotBlank(accessionMCPD.otherNumb)) {
						request.setNames(List.of(accessionMCPD.otherNumb.split(";")));
					}

					// Add other IDs
//				BasePGRFAId basePGRFAIds = new BasePGRFAId();
//				request.setIds(List.of(basePGRFAIds));

					request.setMlsstatus(accessionMCPD.mlsStat);
					request.setHistorical(accessionMCPD.historical != null && accessionMCPD.historical ? "y" : "n");

					Acquisition acquisition = new Acquisition();
					Actor actor = new Actor();
					actor.setWiews(accessionMCPD.donorCode);
					actor.setName(accessionMCPD.donorName);
					acquisition.setProvider(actor);
					acquisition.setSampleid(accessionMCPD.donorNumb);
					request.setAcquisition(acquisition);

					Collection collection = new Collection();
					if (StringUtils.isNotBlank(accessionMCPD.collName)) { // collector name is required
						Collector collector = new Collector();
						collector.setWiews(accessionMCPD.collCode);
						collector.setName(accessionMCPD.collName);
						collector.setAddress(accessionMCPD.collInstAddress);
						collector.setCountry(accessionMCPD.origCty);
						collection.setCollectors(List.of(collector));
					}

					collection.setSampleid(accessionMCPD.collNumb);
					collection.setMissid(accessionMCPD.collMissid);
					collection.setSite(accessionMCPD.collSite);
					if (accessionMCPD.decLatitude != null && accessionMCPD.decLongitude != null) {
						collection.setLat(String.valueOf(accessionMCPD.decLatitude));
						collection.setLon(String.valueOf(accessionMCPD.decLongitude));
						collection.setDatum(accessionMCPD.coordDatum);
						collection.setGeoref(accessionMCPD.geoRefMeth);
						if (accessionMCPD.coordUncert != null) {
							collection.setUncert(String.valueOf(accessionMCPD.coordUncert));
						}
					}
					collection.setElevation(accessionMCPD.elevation);
					if (StringUtils.isNotBlank(accessionMCPD.collDate)) {
						collection.setDate(convertMcpdDateToRequest(accessionMCPD.collDate));
					}
					collection.setSource(accessionMCPD.collSrc == null ? "" : String.valueOf(accessionMCPD.collSrc));
					request.setCollection(collection);

					Breeding breeding = new Breeding();
					Breeder breeder = new Breeder();
					breeder.setWiews(accessionMCPD.bredCode);
					breeder.setName(accessionMCPD.bredName);
					breeding.setBreeders(List.of(breeder));
					breeding.setAncestry(accessionMCPD.ancest);
					request.setBreeding(breeding);

					try {
						if (request instanceof PGRFARegistration) {
							var response = managerApi.registerPGRFA((PGRFARegistration) request);

							// GLIS DOI API returned 200 OK
							assert (response.getDoi() != null);
							assert (response.getSampleid().equals(accession.getAccessionNumber()));
							// Assign DOI
							try {
								var saved = TransactionHelper.executeInTransaction(false, () -> {
									var a = accessionRepository.getReferenceById(accession.getId());
									a.setDoi(response.getDoi());
									return accessionRepository.save(a);
								});
								doiUpdate.doi = saved.getDoi();
								doiUpdate.error = null; // Just in case
								assignDoiState.get().updateProgress(doiUpdate, AssignDoiProgressData.Status.UPLOADING);
							} catch (Throwable e) {
								doiUpdate.error = e.getMessage();
								assignDoiState.get().updateProgress(doiUpdate, AssignDoiProgressData.Status.UPLOADING);
							}

						} else if (request instanceof PGRFAUpdate) {
							var response = managerApi.updatePGRFA((PGRFAUpdate) request);

							// GLIS DOI API returned 200 OK
							assert (response.getDoi() != null);
							assert (response.getSampleid().equals(accession.getAccessionNumber()));
							doiUpdate.error = null; // Just in case
							assignDoiState.get().updateProgress(doiUpdate, AssignDoiProgressData.Status.UPLOADING);
						}

					} catch (HttpClientErrorException e) {
						errorCounter.incrementAndGet();
						log.error("Error calling GLIS API {}: {}", e.getClass(), e.getMessage());

						if (HttpStatus.UNAUTHORIZED.equals(e.getStatusCode())) {
							doiUpdate.error = e.getStatusCode().toString();
						} else if (e.getResponseHeaders().getContentType().isCompatibleWith(MediaType.APPLICATION_XML)) {
							var glisResponse = xmlMapper.readValue(e.getResponseBodyAsString(), PGRFARegistrationResponse.class);
							doiUpdate.error = glisResponse.getError();
						} else {
							doiUpdate.error = e.getMessage();
						}
						assignDoiState.get().updateProgress(doiUpdate, AssignDoiProgressData.Status.UPLOADING);
					} catch (Throwable e) {
						errorCounter.incrementAndGet();
						log.error("Error interacting with GLIS API {}: {}", e.getClass(), e.getMessage());
						doiUpdate.error = e.getMessage();
						assignDoiState.get().updateProgress(doiUpdate, AssignDoiProgressData.Status.UPLOADING);
					}

					if (errorCounter.get() > 10) {
						log.warn("Stopping after too many (10) errors.");
						assignDoiState.get().updateProgress(AssignDoiProgressData.Status.ABORTED, ASSIGN_DOI_TO_MANY_ERRORS);
						return false;
					}
				}
				return true;
			});
		if (completed) {
			assignDoiState.get().updateProgress(AssignDoiProgressData.Status.DONE);
		}
		return assignDoiState.get().progress.processed;
	}

	private ManagerApi createGlisManager() throws Exception {
		var managerApi = managerApiFactory.getObject();
		if (managerApi == null) {
			throw new InvalidApiUsageException("GLIS client not available.");
		}

		if (log.isDebugEnabled() || log.isTraceEnabled()) {
			managerApi.getApiClient().setDebugging(true);
		}

		try {
			String glisUsername = appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_USERNAME).getValue();
			appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_PASSWORD).getValue();
			String glisInstitutePid = appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_PID).getValue();
			appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_INSTITUTE_NAME).getValue();
			appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_INSTITUTE_ADDRESS).getValue();
			appSettingsService.getSetting(APPSETTINGS_ITPGRFA_GLIS, SETTING_INSTITUTE_COUNTRY_CODE).getValue();

			if (StringUtils.isBlank(glisUsername) || StringUtils.isBlank(glisInstitutePid)) {
				throw new InvalidApiUsageException("Missing credentials for GLIS");
			}
			return managerApi;

		} catch (NotFoundElement e) {
			throw new InvalidApiUsageException("GLIS configuration is incomplete", e);
		}
	}

	private String determineMethod(AccessionMCPD accessionMCPD) {
		if (accessionMCPD.collDate != null) {
			return "acqu"; // Acquisition
		} else if (accessionMCPD.donorCode != null || accessionMCPD.donorName != null) {
			return "acqu"; // Acquisition
		} else if (accessionMCPD.bredCode != null || accessionMCPD.bredName != null) {
			return "acqu"; // Acquisition
		} else {
			return "obin"; // Inherited
		}
	}

	private String convertMcpdDateToRequest(String mcpdDate) {
		if (mcpdDate == null) return null;
		String requestDate = null;

		if (mcpdDate.matches("\\d{4}[0-]{4}")) {
			requestDate = mcpdDate.substring(0, 4);
		} else if (mcpdDate.matches("\\d{6}[0-]{2}")) {
			requestDate = String.format("%s-%s", mcpdDate.substring(0, 4), mcpdDate.substring(4, 6));
		} else if (mcpdDate.matches("\\d{8}")) {
			requestDate = String.format("%s-%s-%s", mcpdDate.substring(0, 4), mcpdDate.substring(4, 6), mcpdDate.substring(6, 8));
		}
		return requestDate;
	}
}