RobotsTxtController.java

/*
 * Copyright 2021 Global Crop Diversity Trust
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.gringlobal.mvc;

import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.net.URL;

@RestController
@RequestMapping("/robots.txt")
public class RobotsTxtController implements InitializingBean {

	@Value("${base.url}")
	private String baseUrl;

	private String host;

	@Value("${robots.allow}")
	private boolean allowRobots;

	private static final String ROBOTS_ALLOW_ALL = "User-Agent: *\nAllow: /";
	private static final String ROBOTS_DENY_ALL = "User-Agent: *\nDisallow: /";

	@Override
	public void afterPropertiesSet() throws Exception {
		var siteUrl = new URL(baseUrl);
		host = siteUrl.getHost();
	}

	@GetMapping(value = "")
	public String robotsTxt(HttpServletResponse response, HttpServletRequest request) {
		response.setContentType("text/plain");

		if (!allowRobots) {
			return ROBOTS_DENY_ALL;
		}

		String reqHost = StringUtils.substringBefore(request.getHeader("Host"), ":");

		if (StringUtils.equalsIgnoreCase(this.host, reqHost)) {
			return ROBOTS_ALLOW_ALL;
		} else {
			return ROBOTS_DENY_ALL;
		}
	}
}