<?php
//CONFIGURE ME//
$disallow=array("/password",
	        "/admin",
		"/secret",
		"/.private",
		"/admin/"
		);

$disallow_none = TRUE; 	// Disallow robots that it doesn't recognise with a simple Dissalow: /
						// Probably a good idea, will stop unknown robots crawling pages you want them on(bad),
						// but will stop them going on ones you don't want them on also(good).

$UserAgentCheck = TRUE; //Keep enabled for now, as there is not a sufficient $hosts list

$Additional="#Comment Example\nCrawl-delay: 100\n";

function is_in_range($ip, $start, $end) {
  $ip = ip2long($ip);
  $start = ip2long(start);
  $end = ip2long($end);

  return (($ip >= $range1) && ($ip <= $range2));
}

function clean_file($array)
{
	foreach($array as $key => $entry)
	{
		$entry=trim($entry);
		$array[$key]=$entry;
	}
	
	return $array;
}
											
header('Content-type: text/plain');

//robots listed at http://robotstxt.org, if they're not on the list then they don't get to spider you.
//removed: wget, xget
//these are programs, not real spiders (can be forced to disobey robots.txt)
//added: ia_archiver, msnbot
//I'm sure many more are missing, oh well..
$robots=file('robotlist.txt');
$robots=clean_file($robots);
//hosts list is currently very minimal
$hosts=file('hostlist.txt');

$agent = substr($_SERVER['HTTP_USER_AGENT'],0,strpos($_SERVER['HTTP_USER_AGENT'],"/"));
echo "User-Agent: *\n"; //  using * because the actual User-Agent doesn't seem to be the same for some of these bots.

if($UserAgentCheck==TRUE)
{
	
	if (in_array(strtolower($agent), $robots)) {
	    $robot=TRUE;
	    
	}

}

if($robot!=TRUE)
{
	$userhost=gethostbyaddr($_SERVER['REMOTE_ADDR']);
    foreach($hosts as $host)
    {
	    $userhostcompare=substr($userhost,strpos($userhost,$host),strlen($host));
	    
        if($userhostcompare==$host)
        {
        	$robot=TRUE;
        }
    }
}  

if($robot)
{
	echo $Additional;
    
    foreach($disallow as $dis) {
	    
        echo "Disallow: $dis\n";
    }
}

elseif($disallow_none) {
    //Here you can add a special message to the snoopy person//
     echo "Disallow: /\n"; 
}
?>