RobotsTxtController.java

/*
 * Copyright 2026 Global Crop Diversity Trust
 * Licensed under the Apache License, Version 2.0
 * See LICENSE file in project root folder or http://www.apache.org/licenses/LICENSE-2.0
 */

package org.gringlobal.mvc;

import java.net.URL;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Strings;

@RestController
@RequestMapping("/robots.txt")
public class RobotsTxtController implements InitializingBean {

	@Value("${base.url}")
	private String baseUrl;

	private String host;

	@Value("${robots.allow}")
	private boolean allowRobots;

	private static final String ROBOTS_ALLOW_ALL = "User-Agent: *\nAllow: /";
	private static final String ROBOTS_DENY_ALL = "User-Agent: *\nDisallow: /";

	@Override
	public void afterPropertiesSet() throws Exception {
		var siteUrl = new URL(baseUrl);
		host = siteUrl.getHost();
	}

	@GetMapping(value = "")
	public String robotsTxt(HttpServletResponse response, HttpServletRequest request) {
		response.setContentType("text/plain");

		if (!allowRobots) {
			return ROBOTS_DENY_ALL;
		}

		String reqHost = StringUtils.substringBefore(request.getHeader("Host"), ":");

		if (Strings.CI.equals(this.host, reqHost)) {
			return ROBOTS_ALLOW_ALL;
		} else {
			return ROBOTS_DENY_ALL;
		}
	}
}