#VERSION,2.00
#LASTMOD,11.10.2007

###############################################################################
#  Copyright (C) 2004 CIRT, Inc.
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; version 2
#  of the License only.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
###############################################################################

###############################################################################
# PURPOSE
# Check out the robots.txt file
###############################################################################

sub nikto_robots
{
    (my $RES, $CONTENT) = fetch("/robots.txt", "GET");

    if (($RES eq 200) || ($RES eq $FoF{okay}{response}))    # got one!
    {
        if (is_404("robots.txt", $CONTENT, $RES)) { return; }

        my ($DIRS, $RFILES) = "";
        my $DISCTR = 0;
        my @DOC = split(/\n/, $CONTENT);
        foreach my $line (@DOC)
        {
            $line = char_escape($line);
            if ($line =~ /disallow/i)                       # hmmm...
            {
                chomp($line);
                $line =~ s/\#.*$//;
                $line =~ s/(^\s+|\s+$)//g;
                $line =~ s/\s+/ /g;
                $line =~ s/\\t/ /g;
                $line =~ s/disallow(:)?( )?//i;
                $line =~ s/\*//g;
                $line =~ s/\/+/\//g;

                if ($line eq "") { next; }

                # try to figure out file vs dir... just guess...
                if (($line !~ /\./) && ($line !~ /\/$/)) { $line .= "/"; }

                $line = LW2::uri_normalize($line);

                # figure out dirs/files...
                my $realdir  = LW2::uri_get_dir($line);
                my $realfile = $line;
                $realfile =~ s/^$realdir//;

                nprint("- robots.txt entry dir:$realdir -- file:$realfile", "d");
                if (($realdir  ne "") && ($realdir  ne "/")) { $DIRS{$realdir}++; }
                if (($realfile ne "") && ($realfile ne "/")) { $RFILES{$realfile}++; }
                $DISCTR++;
            }    # end if $line =~ disallow
        }    # end foreach my $line (@DOC)_

        # add them  to mutate dir/file
        foreach my $d (sort keys %DIRS)
        {
            if ($VARIABLES{"\@MUTATEDIRS"} !~ /$d/) { $VARIABLES{"\@MUTATEDIRS"} .= " $d"; }
        }
        foreach my $f (sort keys %RFILES)
        {
            if ($VARIABLES{"\@MUTATEFILES"} !~ /$d/) { $VARIABLES{"\@MUTATEFILES"} .= " $f"; }
        }

        if ($DISCTR eq 1)
        {
            nprint("+ /robots.txt - contains $DISCTR 'disallow' entry which should be manually viewed (added to mutation file lists) (GET).", "", "kb");
        } elsif ($DISCTR > 1)
        {
            nprint("+ /robots.txt - contains $DISCTR 'disallow' entries which should be manually viewed (added to mutation file lists) (GET).", "", "kb");
        } else
        {
            nprint("- /robots.txt - retrieved but it does not contain any 'disallow' entries, which is odd. This should be checked manually.(GET)", "", "kb");
        }
        $TARGETS{$CURRENT_HOST_ID}{total_vulns}++;
    }    # end if $RES eq 200
}

1;