GenesysController.java

/*
 * Copyright 2021 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.gringlobal.api.v1.impl.integration;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonUnwrapped;

import lombok.extern.slf4j.Slf4j;
import lombok.Data;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.genesys.blocks.security.SecurityContextUtil;
import org.genesys.client.GenesysApi;
import org.genesys.client.api.AccessionApi;
import org.genesys.client.api.AccessionUpsertApi;
import org.genesys.client.api.InstituteRequestApi;
import org.genesys.client.invoker.ApiClient;
import org.genesys.client.invoker.auth.HttpBearerAuth;
import org.genesys.client.model.AccessionFilter;
import org.genesys.client.model.AccessionOpResponse;
import org.genesys.client.model.AccessionUpload;
import org.genesys.client.model.AccessionUploadColl;
import org.genesys.client.model.AccessionUploadTaxonomy;
import org.genesys.client.model.FilteredPageAccession;
import org.genesys.client.model.FilteredPageMaterialSubRequest;
import org.genesys.client.model.MaterialSubRequest;
import org.genesys.client.model.MaterialSubRequestFilter;
import org.genesys.client.model.MaterialSubRequestState;
import org.genesys.client.model.ProviderInfoRequest;
import org.genesys.client.model.UpsertResult;
import org.gringlobal.api.exception.InvalidApiUsageException;
import org.gringlobal.api.exception.NotFoundElement;
import org.gringlobal.api.model.AccessionInfo;
import org.gringlobal.api.model.SiteInfo;
import org.gringlobal.api.model.TaxonomySpeciesInfo;
import org.gringlobal.api.v1.ApiBaseController;
import org.gringlobal.api.v1.Pagination;
import org.gringlobal.application.config.IntegrationConfig.GenesysClientFactory;
import org.gringlobal.custom.elasticsearch.SearchException;
import org.gringlobal.model.AppSetting;
import org.gringlobal.model.community.AccessionMCPD;
import org.gringlobal.service.AccessionService;
import org.gringlobal.service.AppSettingsService;
import org.gringlobal.service.ShortFilterService;
import org.gringlobal.worker.AccessionMCPDConverter;
import org.openapitools.jackson.nullable.JsonNullable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import com.github.scribejava.core.builder.ServiceBuilder;
import com.github.scribejava.core.model.OAuth2AccessToken;

import io.swagger.v3.oas.annotations.Operation;
import org.springdoc.api.annotations.ParameterObject;
import io.swagger.v3.oas.annotations.tags.Tag;

import javax.validation.Valid;

/**
 * Integration with Genesys.
 * 
 * @author Matija Obreza
 */
@RestController("genesysApi1")
@RequestMapping(GenesysController.API_URL)
@PreAuthorize("isAuthenticated()")
@Tag(name = "Genesys")
@Slf4j
public class GenesysController extends ApiBaseController {

	/** The Constant API_URL. */
	public static final String API_URL = ApiBaseController.APIv1_BASE + "/genesys";

	public static final String APPSETTING_CLIENT_ID = "clientId";
	public static final String APPSETTING_CLIENT_SECRET = "clientSecret";
	public static final String APPSETTING_INST_CODE = "instCode";
	
	private static final String ACCESSIONS_TO_MANY_ERRORS = "Upload operation was stopped due to many errors while uploading accessions to Genesys.";

	private static final String HEADER_X_GENESYS_AUTH = "X-Genesys-Auth";
	private final AtomicReference<UploadState> uploadState = new AtomicReference<>();

	@Autowired
	private AppSettingsService appSettingsService;

	@Autowired
	private AccessionService accessionService;

	@Autowired
	private ShortFilterService shortFilterService;

	@Autowired
	private ThreadPoolTaskExecutor executor;

	@Autowired(required = false) // not required!
	private GenesysClientFactory genesysClientFactory;

	@PreAuthorize("hasAuthority('GROUP_ADMINS') or @ggceSec.actionAllowed('GenesysUpload', 'READ') or @ggceSec.actionAllowed('GenesysRequests', 'READ')")
	@GetMapping("/config")
	@Operation(description = "Show configuration for Genesys API", summary = "Show configuration")
	public Map<String, String> getConfig() throws Exception {
		String genesysUrl = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, GenesysClientFactory.APPSETTING_GENESYS_URL, String.class).orElse("https://api.genesys-pgr.org");
		String clientId = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, APPSETTING_CLIENT_ID, String.class).orElse("");

		var clientSecret = "";
		if (SecurityContextUtil.hasAuthority("GROUP_ADMINS")) {
			clientSecret = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, APPSETTING_CLIENT_SECRET, String.class).orElse("");
		}

		String origin = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, GenesysClientFactory.APPSETTING_ORIGIN, String.class).orElse("");
		String instCode = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, APPSETTING_INST_CODE, String.class).orElse("");
		String debug = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, GenesysClientFactory.APPSETTING_DEBUG, String.class).orElse("N");

		return Map.of(
			// instCode
			GenesysClientFactory.APPSETTING_DEBUG, debug,
			// URL
			GenesysClientFactory.APPSETTING_GENESYS_URL, genesysUrl,
			// clientId
			APPSETTING_CLIENT_ID, clientId,
			// clientSecret
			APPSETTING_CLIENT_SECRET, clientSecret,
			// origin
			GenesysClientFactory.APPSETTING_ORIGIN, origin, 
			// instCode
			APPSETTING_INST_CODE, instCode
			// .
		);
	}

	@PreAuthorize("hasAuthority('GROUP_ADMINS')")
	@PutMapping("/config")
	@Operation(description = "Set configuration for Genesys API", summary = "Set configuration")
	public Map<String, String> putConfig(@RequestBody(required = true) Map<String, String> config) throws Exception {
		for (String param : List.of(GenesysClientFactory.APPSETTING_DEBUG, GenesysClientFactory.APPSETTING_GENESYS_URL, APPSETTING_CLIENT_ID, APPSETTING_CLIENT_SECRET, GenesysClientFactory.APPSETTING_ORIGIN, APPSETTING_INST_CODE)) {
			AppSetting appSetting = null;
			try { 
				appSetting = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, param);
			} catch (NotFoundElement e) {
				// Setting does not yet exist
			}
			if (appSetting == null) {
				appSetting = new AppSetting(GenesysClientFactory.SETTINGS_GENESYS, param, config.get(param));
				appSettingsService.create(appSetting);
			} else {
				appSetting.setValue(config.get(param));
				appSettingsService.update(appSetting);
			}
		}

		return getConfig();
	}

	@PostMapping("/connect")
	@Operation(description = "Obtain access token for Genesys API", summary = "Connect to Genesys")
	@PreAuthorize("hasAuthority('GROUP_ADMINS') or @ggceSec.actionAllowed('GenesysUpload', 'READ') or @ggceSec.actionAllowed('GenesysRequests', 'READ')")
	public String connect() {
		String genesysApiUrl;
		String clientId;
		String clientSecret;

		try {
			genesysApiUrl = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, GenesysClientFactory.APPSETTING_GENESYS_URL, String.class).get();
			clientId = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, APPSETTING_CLIENT_ID, String.class).get();
			clientSecret = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, APPSETTING_CLIENT_SECRET, String.class).get();
		} catch (NoSuchElementException e) {
			throw new InvalidApiUsageException("Genesys API client is not properly configured", e);
		}
		var serviceBuilder = new ServiceBuilder(clientId).callback("oob");
		if (!StringUtils.isEmpty(clientSecret)) {
			serviceBuilder.apiSecret(clientSecret);
		}
		var service = serviceBuilder.build(new GenesysApi(genesysApiUrl));
		try {
			OAuth2AccessToken clientToken = service.getAccessTokenClientCredentialsGrant();
			return clientToken.getAccessToken();
		} catch (IOException | InterruptedException | ExecutionException e) {
			log.warn("Error getting tokens from {}: {}", genesysApiUrl, e.getMessage());
			throw new InvalidApiUsageException("Could not authenticate against " + genesysApiUrl + ": " + e.getMessage(), e);
		}
	}

	@PostMapping("/requests")
	@Operation(description = "Fetch request data from Genesys", summary = "List requests")
	@PreAuthorize("@ggceSec.actionAllowed('GenesysRequests', 'READ')")
	public FilteredPageMaterialSubRequest listRequests(@RequestHeader(name = HEADER_X_GENESYS_AUTH, required = true) String bearerToken, @ParameterObject final Pagination page,
		@RequestBody(required = false) MaterialSubRequestFilter filter) throws Exception {
		ApiClient genesysClient = genesysClientFactory.getObject();
		String instCode = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, APPSETTING_INST_CODE, String.class).get();
		log.warn("Checking requests for {}", instCode);
		HttpBearerAuth jwtAuth = (HttpBearerAuth) genesysClient.getAuthentication("JWT");
		jwtAuth.setBearerToken(bearerToken);

		var requestApi = new InstituteRequestApi(genesysClient);

		var sort = page.getS() != null && page.getS().length > 0 ? page.getS()[0] : "id";
		var direction = page.getD() != null && page.getD().length > 0 ? page.getD()[0] : Sort.Direction.DESC;

		if (filter == null) {
			filter = new MaterialSubRequestFilter();
		}

		// Excluding DRAFT(-1) state
		if (filter.getNOT() != null) {
			filter.getNOT().addStateItem(MaterialSubRequestState.GENESYS_SUBREQUEST_STATUS_DRAFT);
		} else {
			filter.NOT(new MaterialSubRequestFilter().state(Set.of(MaterialSubRequestState.GENESYS_SUBREQUEST_STATUS_DRAFT)));
		}

		return requestApi.listInstituteRequests(instCode, page.getP(), page.getL(), sort, direction.name(), filter);
	}

	@PostMapping("/requests/{uuid:.{36}}/provider/info")
	@Operation(description = "Update MaterialSubRequest provider info on Genesys", summary = "Update subrequest provider info")
	@PreAuthorize("@ggceSec.actionAllowed('GenesysRequests', 'WRITE')")
	public MaterialSubRequest updateRequestProvider(@RequestHeader(name = HEADER_X_GENESYS_AUTH, required = true) String bearerToken,
		@PathVariable("uuid") UUID uuid, @RequestBody @Valid ProviderInfoRequest providerInfo) throws Exception {
		
		ApiClient genesysClient = genesysClientFactory.getObject();
		HttpBearerAuth jwtAuth = (HttpBearerAuth) genesysClient.getAuthentication("JWT");
		jwtAuth.setBearerToken(bearerToken);
		var requestApi = new InstituteRequestApi(genesysClient);

		return requestApi.updateRequestProviderInfo(uuid, providerInfo);
	}

	@PostMapping("/accessions")
	@Operation(description = "Fetch passport data from Genesys", summary = "List accessions")
	@PreAuthorize("@ggceSec.actionAllowed('GenesysRequests', 'READ')")
	public FilteredPageAccession listAccessions(@RequestHeader(name = HEADER_X_GENESYS_AUTH, required = true) final String bearerToken,
			@RequestBody(required = false) AccessionFilter filter, @ParameterObject final Pagination page) throws Exception {
		ApiClient genesysClient = genesysClientFactory.getObject();
		String instCode = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, APPSETTING_INST_CODE, String.class).get();
		log.warn("Checking requests for {}", instCode);

		HttpBearerAuth jwtAuth = (HttpBearerAuth) genesysClient.getAuthentication("JWT");
		jwtAuth.setBearerToken(bearerToken);

		var accessionApi = new AccessionApi(genesysClient);

		if (filter == null)
			filter = new AccessionFilter();

		return accessionApi.listAccessions(page.getP(), page.getL(), filter);
	}

	@PostMapping("/accessions/cancel-upload")
	@PreAuthorize("@ggceSec.actionAllowed('GenesysUpload', 'WRITE')")
	public ResponseEntity<HttpStatus> cancelUploadAccessions(@RequestParam(name = "f", required = true) final String filterCode) {

		if (uploadState.get() != null && uploadState.get().filterCode.equals(filterCode)) {
			// cancel only active uploading
			if (uploadState.get().progress.status == UploadProgressData.Status.UPLOADING) {
				var sid = SecurityContextUtil.getCurrentUser() != null ? SecurityContextUtil.getCurrentUser().getSid() : null;
				log.warn("Upload for filter code={} was canceled by {}", filterCode, sid);
				uploadState.set(uploadState.get().cancel());
			}
		} else {
			throw new InvalidApiUsageException("No active uploads by filter code=" + filterCode + " is currently running.");
		}
		return ResponseEntity.ok().build();
	}

	@GetMapping("/accessions/available-fields")
	public Map<String, Collection<String>> getFieldInfo() {
		return Map.of("excluded", getNeverUploadFields(), "all", FIELDS);
	}

	@PostMapping("/accessions/upload")
	@PreAuthorize("@ggceSec.actionAllowed('GenesysUpload', 'WRITE')")
	public UploadProgressData uploadAccessions(@RequestHeader(name = HEADER_X_GENESYS_AUTH) final String bearerToken,
			@RequestParam(name = "f", required = false) String filterCode,
			@RequestParam(name = "uploadOnly", required = false) Set<String> fieldsToUpload,
			@RequestBody(required = false) final org.gringlobal.service.filter.AccessionFilter filter
	) throws Exception {

		final var filterInfo = shortFilterService.processFilter(filterCode, filter, org.gringlobal.service.filter.AccessionFilter.class);
		if (filterInfo.filter.toString().equals("{}"))
			throw new InvalidApiUsageException("Refusing to upload accessions by empty filter.");

		if (uploadState.get() != null && uploadState.get().filterCode.equals(filterInfo.filterCode)) {
			if (uploadState.get().progress.status != UploadProgressData.Status.UPLOADING) {
				// return progress if it's DONE or ABORTED, then clear it
				return uploadState.getAndSet(null).progress;
			} else {
				return uploadState.get().progress; // return progress, still uploading...
			}
		} else if (uploadState.get() != null && !uploadState.get().filterCode.equals(filterInfo.filterCode)) {
			if (uploadState.get().progress.status == UploadProgressData.Status.UPLOADING) {
				// only one active upload at a time!
				throw new InvalidApiUsageException("Another upload for code=" + uploadState.get().filterCode + " is currently running.");
			}
		}

		final ApiClient genesysClient = genesysClientFactory.getObject();
		final HttpBearerAuth jwtAuth = (HttpBearerAuth) genesysClient.getAuthentication("JWT");
		jwtAuth.setBearerToken(bearerToken);

		executor.execute(() -> {
			uploadState.set(new UploadState(filterInfo.filterCode, new UploadProgressData(0L, 0L, UploadProgressData.Status.UPLOADING)));
			try {
				upsert(filterInfo, fieldsToUpload, genesysClient);
			} catch (Throwable e) {
				log.warn("Error while uploading accessions: {}", e.getMessage());
				if (CollectionUtils.isEmpty(uploadState.get().progress.errors)) {
					uploadState.get().updateProgress(0, 0, UploadProgressData.Status.ABORTED, e.getLocalizedMessage());
				}
			}
		});
		return new UploadProgressData(0, 0, UploadProgressData.Status.UPLOADING);
	}

	private void upsert(ShortFilterService.FilterInfo<org.gringlobal.service.filter.AccessionFilter> filterInfo, Set<String> fieldsToUpload, ApiClient genesysClient) throws SearchException {
		final var accessionUpsertApi = new AccessionUpsertApi(genesysClient);

		Set<String> neverUploadFields = getNeverUploadFields();
		if (fieldsToUpload != null) {
			log.warn("Only the following fields will be updated: {}", fieldsToUpload);
		}

		Pageable pageable = PageRequest.of(0, 50);
		var toUpsert = accessionService.listMCPD(filterInfo.filter, pageable);

		final Map<String, Set<AccessionMCPD>> accessionsGroupedByInst = new HashMap<>();

		AtomicReference<Long> uploaded = new AtomicReference<>(0L);
		long total = toUpsert.getTotalElements();

		// Update total
		uploadState.get().updateProgress(total, uploaded.get(), UploadProgressData.Status.UPLOADING);

		List<UploadProgressData.ProgressError> progressErrors = new ArrayList<>();

		while (toUpsert.hasContent()) {

			// Group accessions by INSTCODE
			toUpsert.stream().filter(a -> StringUtils.isNotBlank(a.instCode)).forEach(a -> {
				Set<AccessionMCPD> accessions = accessionsGroupedByInst.getOrDefault(a.instCode, new HashSet<>());
				accessions.add(a);
				accessionsGroupedByInst.put(a.instCode, accessions);
			});

			// Process batches by INSTCODE
			for (var entry: accessionsGroupedByInst.entrySet()) {
				// check the status before each upsert
				if (uploadState.get().canceled) {
					throw new InvalidApiUsageException("Uploading has been canceled by the user.");
				}

				String instCode = entry.getKey();
				var updates = entry.getValue().stream()
					// Convert AccessionMCPD for Genesys
					.map(a -> accessionToJson(fieldsToUpload, a))
					// Set fields we never upload to null
					.peek(a -> neverUpload(neverUploadFields, a))
					// Collect
					.collect(Collectors.toList());

				var accessionMCPDMap = entry.getValue().stream().collect(Collectors.toMap(mcpd -> mcpd.acceNumb, mcpd -> mcpd));
				log.debug("Prepared batch of {} accessions for INSTCODE={}", updates.size(), instCode);

				// Upload batch for this INSTCODE
				if (updates.size() > 0) {
					var uploadOps = accessionUpsertApi.upsertInstituteAccessions(instCode, updates);
					uploadOps.forEach(op -> {
						log.trace("Upload resulted in: {} instCode={} doi={} acceNumb={} genus={} error={}", op.getResult().getAction(), op.getInstCode(), op.getDoi(), op.getAcceNumb(), op.getGenus(), op.getError());
						if (op.getResult().getAction() == UpsertResult.ActionEnum.ERROR) {
							progressErrors.add(new UploadProgressData.ProgressError(op, accessionMCPDMap.get(op.getAcceNumb())));
							// If more than 10 errors, stop uploading, report error
							if (progressErrors.size() > 10) {
								uploadState.get().updateProgress(total, uploaded.get(), UploadProgressData.Status.ABORTED, ACCESSIONS_TO_MANY_ERRORS, progressErrors);
								throw new InvalidApiUsageException("Error treshold reached, upload to Genesys aborted.");
							}
						}
					});
					uploaded.set(uploaded.get() + uploadOps.size());
					uploadState.get().updateProgress(total, uploaded.get(), UploadProgressData.Status.UPLOADING, null, progressErrors);
				}
			}

			// Clear
			accessionsGroupedByInst.clear();
			// Load next page
			pageable = pageable.next();
			toUpsert = accessionService.listMCPD(filterInfo.filter, pageable);
		}

		uploadState.get().updateProgress(total, uploaded.get() - progressErrors.size(), UploadProgressData.Status.DONE, null, progressErrors);
	}

	/**
	 * AppSetting {@code neverUploadFields} in category {@code GENESYS_CLIENT}
	 * can contain a comma separated list of fields that must not be uploaded to Genesys.
	 */
	private Set<String> getNeverUploadFields() {
		String neverUploadString = appSettingsService.getSetting(GenesysClientFactory.SETTINGS_GENESYS, GenesysClientFactory.APPSETTING_NEVER_UPLOAD, String.class).orElse("");
		if (StringUtils.isNotBlank(neverUploadString)) {
			log.warn("Never upload: {}", neverUploadString);
			var parsed = Arrays.stream(neverUploadString.split("\\s*,\\s*")).map(StringUtils::trimToNull).filter(Objects::nonNull).collect(Collectors.toSet());
			if (parsed.size() > 0) {
				log.warn("Data will be removed from Genesys for: {}", parsed);
				return parsed;
			}
		}
		return Set.of();
	}

	/**
	 * Prepare AccessionMCPD for Genesys. If {@code fieldsToUpload} is provided then fields listed
	 * will be mapped to {@code undefined} which leaves data in Genesys untouched.
	 * 
	 * @param fieldsToUpload Set of MCPD field names to include in the upload. Can be null to include all.
	 * @param a the Accession in MCPD
	 * @return JSON for upload
	 */
	public static AccessionUpload accessionToJson(Set<String> fieldsToUpload, AccessionMCPD a) {
		AccessionUpload json = new AccessionUpload();
		// Collecting data
		var coll = new AccessionUploadColl();
		json.setColl(coll);
		// Taxonomy
		var taxonomy = new AccessionUploadTaxonomy();
		json.setTaxonomy(taxonomy);

		// MCPD
		json.doi(a.puid); // 0. Persistent unique identifier
		json.instituteCode(a.instCode); // 1. Institute code
		json.accessionNumber(a.acceNumb); // 2. Accession number

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_collNumb)) {
			coll.collNumb(a.collNumb); // 3. Collecting number
		} else {
			coll.setCollNumb_JsonNullable(JsonNullable.undefined());
		}

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_collCode)) {
			coll.collCode(AccessionMCPDConverter.streamSplit(";", a.collCode).collect(Collectors.toSet())); // 4. Collecting institute code
		} else {
			coll.setCollCode_JsonNullable(JsonNullable.undefined());
		}

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_collName)) {
			coll.collName(AccessionMCPDConverter.streamSplit(";", a.collName).collect(Collectors.toSet())); // 4.1 Collecting institute name
		} else {
			coll.setCollName_JsonNullable(JsonNullable.undefined());
		}

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_collInstAddress)) {
			coll.collInstAddress(AccessionMCPDConverter.streamSplit(";", a.collInstAddress).collect(Collectors.toSet())); // 4.1.1 Collecting institute address
		} else {
			coll.setCollInstAddress_JsonNullable(JsonNullable.undefined());
		}

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_collMissId)) {
			coll.collMissId(a.collMissid); // 4.2 Collecting mission identifier
		} else {
			coll.setCollMissId_JsonNullable(JsonNullable.undefined());
		}
	
		taxonomy.genus(a.genus); // 5. Genus

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_species)) {
			taxonomy.species(a.species); // 6. Species
		} else {
			taxonomy.setSpecies_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_spAuthor)) {
			taxonomy.spAuthor(a.spAuthor); // 7. Species authority
		} else {
			taxonomy.setSpAuthor_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_subtaxa)) {
			taxonomy.subtaxa(a.subtaxa); // 8. Subtaxon
		} else {
			taxonomy.setSubtaxa_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_subtAuthor)) {
			taxonomy.subtAuthor(a.subtAuthor); // 9. Subtaxon authority
		} else {
			taxonomy.setSubtAuthor_JsonNullable(JsonNullable.undefined());
		}

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_cropName)) {
			json.cropName(a.cropName); // 10. Common crop name
		} else {
			json.setCropName_JsonNullable(JsonNullable.undefined());
		}

		// 11. Accession name
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_acceName)) {
			if (StringUtils.isNotBlank(a.acceName)) {
				json.addAcceNameItem(a.acceName);
			} else {
				json.acceName(new HashSet<>());
			}
		} else {
			json.setAcceName_JsonNullable(JsonNullable.undefined());
		}

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_acquisitionDate)) {
			json.acquisitionDate(a.acqDate); // 12. Acquisition date
		} else {
			json.setAcquisitionDate_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_origCty)) {
			json.origCty(a.origCty); // 13. Country of origin
		} else {
			json.setOrigCty_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_collSite)) {
			coll.collSite(a.collSite); // 14. Location of collecting site
		} else {
			coll.setCollSite_JsonNullable(JsonNullable.undefined());
		}

		// 15. Geographical coordinates
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_decLatitude)) {
			json.latitude(a.decLatitude); // 15.1 Latitude of collecting site
		} else {
			json.setLatitude_JsonNullable(JsonNullable.undefined());
		}
		// Obsolete // 15.2 Latitude of collecting site (Degrees, Minutes, Seconds format)
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_decLongitude)) {
			json.longitude(a.decLongitude); // 15.3 Longitude of collecting site
		} else {
			json.setLongitude_JsonNullable(JsonNullable.undefined());
		}
		// Obsolete // 15.4 Longitude of collecting site (Degrees, Minutes, Seconds format)
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_coordUncert)) {
			json.coordinateUncertainty(a.coordUncert == null ? null : a.coordUncert.doubleValue()); // 15.5 Coordinate uncertainty
		} else {
			json.setCoordinateUncertainty_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_coordDatum)) {
			json.coordinateDatum(a.coordDatum); // 15.6 Coordinate datum
		} else {
			json.setCoordinateDatum_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_geoRefMeth)) {
			json.georeferenceMethod(a.geoRefMeth); // 15.7 Georeferencing method
		} else {
			json.setGeoreferenceMethod_JsonNullable(JsonNullable.undefined());
		}
		
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_elevation)) {
			json.elevation(a.elevation == null ? null : a.elevation.doubleValue()); // 16. Elevation of collecting site
		} else {
			json.setElevation_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_collDate)) {
			coll.collDate(a.collDate); // 17. Collecting date of sample
		} else {
			coll.setCollDate_JsonNullable(JsonNullable.undefined());
		}
		
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_bredCode)) {
			json.breederCode(AccessionMCPDConverter.streamSplit(";", a.bredCode).collect(Collectors.toSet())); // 18. Breeding institute code
		} else {
			json.setBreederCode_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_bredName)) {
			json.breederName(AccessionMCPDConverter.streamSplit(";", a.bredName).collect(Collectors.toSet())); // 18.1  Breeding institute name
		} else {
			json.setBreederName_JsonNullable(JsonNullable.undefined());
		}

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_sampStat)) {
			json.sampStat(a.sampStat); // 19. Biological status of accession
		} else {
			json.setSampStat_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_ancest)) {
			json.ancest(a.ancest); // 20. Ancestral data
		} else {
			json.setAncest_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_collSrc)) {
			coll.collSrc(a.collSrc); // 21. Collecting/acquisition source
		} else {
			coll.setCollSrc_JsonNullable(JsonNullable.undefined());
		}
		
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_donorCode)) {
			json.donorCode(a.donorCode); // 22. Donor institute code
		} else {
			json.setDonorCode_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_donorName)) {
			json.donorName(a.donorName); // 22.1  Donor institute name
		} else {
			json.setDonorName_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_donorNumb)) {
			json.donorNumb(a.donorNumb); // 23. Donor accession number
		} else {
			json.setDonorNumb_JsonNullable(JsonNullable.undefined());
		}
		
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_otherNumb)) {
			json.otherNumb(AccessionMCPDConverter.streamSplit(";", a.otherNumb).collect(Collectors.toSet())); // 24. Other identifiers associated with the accession
		} else {
			json.setOtherNumb_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_duplSite)) {
			json.duplSite(AccessionMCPDConverter.streamSplit(";", a.duplSite).collect(Collectors.toSet())); // 25. Location of safety duplicates
		} else {
			json.setDuplSite_JsonNullable(JsonNullable.undefined());
		}
		// Unsupported // 25.1  Institute maintaining safety duplicates
		
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_storage)) {
			json.storage(AccessionMCPDConverter.streamSplit(";", a.storage).map(val -> Integer.parseInt(val)).collect(Collectors.toSet())); // 26. Type of germplasm storage
		} else {
			json.setStorage_JsonNullable(JsonNullable.undefined());
		}
		
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_mlsStat)) {
			json.mlsStatus(a.mlsStat == null ? null : 1 == a.mlsStat); // 27. MLS status of the accession
		} else {
			json.setMlsStatus_JsonNullable(JsonNullable.undefined());
		}

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_remarks)) {
			json.remarks(StringUtils.isBlank(a.remarks) ? null : List.of(a.remarks)); // 28. Remarks
		} else {
			json.setMlsStatus_JsonNullable(JsonNullable.undefined());
		}

		// -------

		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_acceUrl)) {
			json.acceUrl(a.acceUrl);
		} else {
			json.setAcceUrl_JsonNullable(JsonNullable.undefined());
		}

		// Genesys specific
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_historical)) {
			json.historic(a.historical); // Status of the accession in the collection
		} else {
			json.setHistoric_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_availability)) {
			json.available(a.availability); // Availability of accession for distribution
		} else {
			json.setAvailable_JsonNullable(JsonNullable.undefined());
		}
		if (fieldsToUpload == null || fieldsToUpload.contains(FIELD_curationType)) {
			json.curationType(a.curationType);
		} else {
			json.setCurationType_JsonNullable(JsonNullable.undefined());
		}

		// json.dataProviderId(a.id);
		// json.acquisitionSource(a.)

		return json;
	}

	/**
	 * Fields that are never uploaded to Genesys are set to {@code null} instead of {@code undefined}.
	 *
	 * @param fieldsToNeverUpload Set of fields to never send to Genesys
	 * @param json Upload object to clean up
	 * @return The incoming object with fields to never upload set to {@code null}
	 */
	public static AccessionUpload neverUpload(Set<String> fieldsToNeverUpload, AccessionUpload json) {
		if (json == null) return json;
		if (CollectionUtils.isEmpty(fieldsToNeverUpload)) return json;

		var coll = json.getColl();
		var taxonomy = json.getTaxonomy();

		if (fieldsToNeverUpload.contains(FIELD_collNumb)) {
			coll.collNumb(null); // 3. Collecting number
		}
		if (fieldsToNeverUpload.contains(FIELD_collCode)) {
			coll.collCode(null); // 4. Collecting institute code
		}
		if (fieldsToNeverUpload.contains(FIELD_collName)) {
			coll.collName(null); // 4.1 Collecting institute name
		}
		if (fieldsToNeverUpload.contains(FIELD_collInstAddress)) {
			coll.collInstAddress(null); // 4.1.1 Collecting institute address
		}
		if (fieldsToNeverUpload.contains(FIELD_collMissId)) {
			coll.collMissId(null); // 4.2 Collecting mission identifier
		}
		if (fieldsToNeverUpload.contains(FIELD_species)) {
			taxonomy.species(null); // 6. Species
		}
		if (fieldsToNeverUpload.contains(FIELD_spAuthor)) {
			taxonomy.spAuthor(null); // 7. Species authority
		}
		if (fieldsToNeverUpload.contains(FIELD_subtaxa)) {
			taxonomy.subtaxa(null); // 8. Subtaxon
		}
		if (fieldsToNeverUpload.contains(FIELD_subtAuthor)) {
			taxonomy.subtAuthor(null); // 9. Subtaxon authority
		}
		if (fieldsToNeverUpload.contains(FIELD_cropName)) {
			json.cropName(null); // 10. Common crop name
		}
		if (fieldsToNeverUpload.contains(FIELD_acceName)) {
			json.acceName(null); // 10. Common crop name
		}
		if (fieldsToNeverUpload.contains(FIELD_acquisitionDate)) {
			json.acquisitionDate(null); // 12. Acquisition date
		}
		if (fieldsToNeverUpload.contains(FIELD_origCty)) {
			json.origCty(null); // 13. Country of origin
		}
		if (fieldsToNeverUpload.contains(FIELD_collSite)) {
			coll.collSite(null); // 14. Location of collecting site
		}
		if (fieldsToNeverUpload.contains(FIELD_decLatitude)) {
			json.latitude(null); // 15.1 Latitude of collecting site
		}
		if (fieldsToNeverUpload.contains(FIELD_decLongitude)) {
			json.longitude(null); // 15.3 Longitude of collecting site
		}
		if (fieldsToNeverUpload.contains(FIELD_coordUncert)) {
			json.coordinateUncertainty(null); // 15.5 Coordinate uncertainty
		}
		if (fieldsToNeverUpload.contains(FIELD_coordDatum)) {
			json.coordinateDatum(null); // 15.6 Coordinate datum
		}
		if (fieldsToNeverUpload.contains(FIELD_geoRefMeth)) {
			json.georeferenceMethod(null); // 15.7 Georeferencing method
		}
		if (fieldsToNeverUpload.contains(FIELD_elevation)) {
			json.elevation(null); // 16. Elevation of collecting site
		}
		if (fieldsToNeverUpload.contains(FIELD_collDate)) {
			coll.collDate(null); // 17. Collecting date of sample
		}
		if (fieldsToNeverUpload.contains(FIELD_bredCode)) {
			json.breederCode(null); // 18. Breeding institute code
		}
		if (fieldsToNeverUpload.contains(FIELD_bredName)) {
			json.breederName(null); // 18.1  Breeding institute name
		}
		if (fieldsToNeverUpload.contains(FIELD_sampStat)) {
			json.sampStat(null); // 19. Biological status of accession
		}
		if (fieldsToNeverUpload.contains(FIELD_ancest)) {
			json.ancest(null); // 20. Ancestral data
		}
		if (fieldsToNeverUpload.contains(FIELD_collSrc)) {
			coll.collSrc(null); // 21. Collecting/acquisition source
		}
		if (fieldsToNeverUpload.contains(FIELD_donorCode)) {
			json.donorCode(null); // 22. Donor institute code
		}
		if (fieldsToNeverUpload.contains(FIELD_donorName)) {
			json.donorName(null); // 22.1  Donor institute name
		}
		if (fieldsToNeverUpload.contains(FIELD_donorNumb)) {
			json.donorNumb(null); // 23. Donor accession number
		}
		if (fieldsToNeverUpload.contains(FIELD_otherNumb)) {
			json.otherNumb(null); // 24. Other identifiers associated with the accession
		}
		if (fieldsToNeverUpload.contains(FIELD_duplSite)) {
			json.duplSite(null); // 25. Location of safety duplicates
		}
		if (fieldsToNeverUpload.contains(FIELD_storage)) {
			json.storage(null); // 26. Type of germplasm storage
		}
		if (fieldsToNeverUpload.contains(FIELD_mlsStat)) {
			json.mlsStatus(null); // 27. MLS status of the accession
		}
		if (fieldsToNeverUpload.contains(FIELD_remarks)) {
			json.remarks(null); // 28. Remarks
		}
		if (fieldsToNeverUpload.contains(FIELD_acceUrl)) {
			json.acceUrl(null);
		}
		if (fieldsToNeverUpload.contains(FIELD_historical)) {
			json.historic(null); // Status of the accession in the collection
		}
		if (fieldsToNeverUpload.contains(FIELD_availability)) {
			json.available(null); // Availability of accession for distribution
		}
		if (fieldsToNeverUpload.contains(FIELD_curationType)) {
			json.curationType(null);
		}
		return json;
	}


	private static class UploadState {
		public String filterCode;
		public UploadProgressData progress;
		public boolean canceled = false;

		public UploadState(String filterCode, UploadProgressData progress) {
			this.filterCode = filterCode;
			this.progress = progress;
		}

		public UploadState updateProgress(long total, long uploaded, UploadProgressData.Status status) {
			this.progress.status = status;
			this.progress.total = total;
			this.progress.uploaded = uploaded;
			return this;
		}
		
		public UploadState updateProgress(long total, long uploaded, UploadProgressData.Status status, String error) {
			this.updateProgress(total, uploaded, status);
			this.progress.errorMessage = error;
			return this;
		}

		public UploadState updateProgress(long total, long uploaded, UploadProgressData.Status status, String error, List<UploadProgressData.ProgressError> errors) {
			this.updateProgress(total, uploaded, status, error);
			this.progress.errors = errors;
			return this;
		}

		public UploadState cancel() {
			this.canceled = true;
			return this;
		}
	}

	@Data
	public static class UploadProgressData {
		public enum Status { UPLOADING, ABORTED, DONE }
		@Data
		public static class ProgressError {
			@JsonUnwrapped
			private AccessionInfo accession;
			private String message;

			ProgressError(AccessionOpResponse op, AccessionMCPD mcpd) {
				var info = new AccessionInfo();
				info.setId(mcpd.id);
				info.setAccessionNumber(op.getAcceNumb());
				info.setDoi(op.getDoi());
				var taxonomy = new TaxonomySpeciesInfo();
				taxonomy.setGenusName(op.getGenus());
				info.setTaxonomySpecies(taxonomy);
				var site = new SiteInfo();
				site.setFaoInstituteNumber(op.getInstCode());
				info.setSite(site);
				this.accession = info;
				this.message = op.getError();
			}
		}

		private Status status;
		private long total;
		private long uploaded;

		@JsonInclude(JsonInclude.Include.NON_NULL)
		public String errorMessage;
		@JsonInclude(JsonInclude.Include.NON_EMPTY)
		private List<ProgressError> errors;

		public UploadProgressData(long total, long uploaded, Status status) {
			this.status = status;
			this.total = total;
			this.uploaded = uploaded;
		}

		public float getProgress() {
			return 100.0f * (uploaded == 0 ? 0 : ((float) (uploaded)) / total);
		}
	}

	public static final String FIELD_collNumb = "collNumb";
	public static final String FIELD_collCode = "collCode";
	public static final String FIELD_collName = "collName";
	public static final String FIELD_collInstAddress = "collInstAddress";
	public static final String FIELD_collMissId = "collMissId";
	public static final String FIELD_species = "species";
	public static final String FIELD_spAuthor = "spAuthor";
	public static final String FIELD_subtaxa = "subtaxa";
	public static final String FIELD_subtAuthor = "subtAuthor";
	public static final String FIELD_cropName = "cropName";
	public static final String FIELD_acceName = "acceName";
	public static final String FIELD_acquisitionDate = "acquisitionDate";
	public static final String FIELD_origCty = "origCty";
	public static final String FIELD_collSite = "collSite";
	public static final String FIELD_decLatitude = "decLatitude";
	public static final String FIELD_decLongitude = "decLongitude";
	public static final String FIELD_coordUncert = "coordUncert";
	public static final String FIELD_coordDatum = "coordDatum";
	public static final String FIELD_geoRefMeth = "geoRefMeth";
	public static final String FIELD_elevation = "elevation";
	public static final String FIELD_collDate = "collDate";
	public static final String FIELD_bredCode = "bredCode";
	public static final String FIELD_bredName = "bredName";
	public static final String FIELD_sampStat = "sampStat";
	public static final String FIELD_ancest = "ancest";
	public static final String FIELD_collSrc = "collSrc";
	public static final String FIELD_donorCode = "donorCode";
	public static final String FIELD_donorName = "donorName";
	public static final String FIELD_donorNumb = "donorNumb";
	public static final String FIELD_otherNumb = "otherNumb";
	public static final String FIELD_duplSite = "duplSite";
	public static final String FIELD_storage = "storage";
	public static final String FIELD_mlsStat = "mlsStat";
	public static final String FIELD_remarks = "remarks";
	public static final String FIELD_acceUrl = "acceUrl";
	public static final String FIELD_historical = "historical";
	public static final String FIELD_availability = "availability";
	public static final String FIELD_curationType = "curationType";

	public static final List<String> FIELDS = List.of(
		FIELD_collNumb,
		FIELD_collCode,
		FIELD_collName,
		FIELD_collInstAddress,
		FIELD_collMissId,
		FIELD_species,
		FIELD_spAuthor,
		FIELD_subtaxa,
		FIELD_subtAuthor,
		FIELD_cropName,
		FIELD_acceName,
		FIELD_acquisitionDate,
		FIELD_origCty,
		FIELD_collSite,
		FIELD_decLatitude,
		FIELD_decLongitude,
		FIELD_coordUncert,
		FIELD_coordDatum,
		FIELD_geoRefMeth,
		FIELD_elevation,
		FIELD_collDate,
		FIELD_bredCode,
		FIELD_bredName,
		FIELD_sampStat,
		FIELD_ancest,
		FIELD_collSrc,
		FIELD_donorCode,
		FIELD_donorName,
		FIELD_donorNumb,
		FIELD_otherNumb,
		FIELD_duplSite,
		FIELD_storage,
		FIELD_mlsStat,
		FIELD_remarks,
		FIELD_acceUrl,
		FIELD_historical,
		FIELD_availability,
		FIELD_curationType
	);
}