Support multiple robots.txt in a single Sitefinity instance

Global.asax.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.Routing;
using System.Web.Security;
using System.Web.SessionState;
using Telerik.Sitefinity.Abstractions;
using Telerik.Sitefinity.Data;
using System.Web.Mvc;

namespace SitefinityWebApp
{
    public class Global : System.Web.HttpApplication
    {

        protected void Application_Start(object sender, EventArgs e)
        {
            Bootstrapper.Initialized += new EventHandler<ExecutedEventArgs>(Bootstrapper_Initialized);
        }

        void Bootstrapper_Initialized(object sender, Telerik.Sitefinity.Data.ExecutedEventArgs e)
        {
            if (e.CommandName == "RegisterRoutes")
            {
                RegisterRoutes();
            }
        }

        public static void RegisterRoutes()
        {
            RouteTable.Routes.MapRoute(
               "Robots.txt",
               "robots.txt",
               new { controller = "RobotsText", action = "Index" },
               new string[] { "SitefinityWebApp.Mvc.Controllers" }
            );
        }

    }
}

RobotsTextController.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Web;
using System.Web.Mvc;

namespace SitefinityWebApp.Mvc.Controllers
{
    public class RobotsTextController : Controller
    {
        //[OutputCache(Duration = 86400)]
        public ActionResult Index()
        {
            // the robots.txt files can be organized by naming them with the host
            // example: quantum.sitefinity.com.txt
            var fileName = String.Format("{0}.txt", Request.Url.Host);
            
            // create a folder named "RobotsText" off the root of the project and store the
            // robots.txt files there
            var filePath = Server.MapPath(String.Format("~/RobotsText/{0}", fileName));

            if (System.IO.File.Exists(filePath))
            {
                var stringBuilder = new StringBuilder();

                stringBuilder.AppendLine(System.IO.File.ReadAllText(filePath));
                stringBuilder.AppendLine(GetSitemapUrl());

                return this.Content(stringBuilder.ToString(), "text/plain", Encoding.UTF8);
            }

            return HttpNotFound();
        }

        // find and return a static result
        public string GetStaticRobotsText()
        {
            var fileName = String.Format("{0}.txt", Request.Url.Host);
            var filePath = Server.MapPath(String.Format("~/RobotsText/{0}", fileName));

            if (System.IO.File.Exists(filePath))
            {
                return System.IO.File.ReadAllText(filePath);
            }

            return string.Empty;
        }

        // dynamically generate the sitemap url
        public string GetSitemapUrl()
        {
            var leftPart = Request.Url.GetLeftPart(UriPartial.Authority);
            
            return String.Format("Sitemap: {0}/sitemap/sitemap-index.xml", leftPart);
        }
    }
}

web.config

<!-- add this to the web.config -->
<configuration>
    <!-- ... -->
    <system.webServer>
        <!-- ... -->
        <handlers>
            <!-- ... -->
            <remove name="Telerik_Web_UI_DialogHandler_aspx" />
            <add name="RobotsText" verb="GET" path="robots.txt" preCondition="integratedMode,runtimeVersionv4.0" type="System.Web.Handlers.TransferRequestHandler" />
            <!-- ... -->
        </handlers>
    </system.webServer>
    <!-- ... -->
</configuration>