http-useragent-tester
NSE detection rule by George Chatzisofroniou · Same as Nmap--See https://nmap.org/book/man-legal.html
Checks if various crawling utilities are allowed by the host.
description = [[
Checks if various crawling utilities are allowed by the host.
]]
---
-- @usage nmap -p80 --script http-useragent-tester.nse <host>
--
-- This script sets various User-Agent headers that are used by different
-- utilities and crawling libraries (for example CURL or wget). If the request is
-- redirected to a page different than a (valid) browser request would be, that
-- means that this utility is banned.
--
-- @args http-useragent-tester.useragents A table with more User-Agent headers.
-- Default: nil
--
-- @output
-- PORT STATE SERVICE REASON
-- 80/tcp open http syn-ack
-- | http-useragent-tester:
-- | Status for browser useragent: 200
-- | Redirected To: https://www.example.com/
-- | Allowed User Agents:
-- | Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)
-- | libwww
-- | lwp-trivial
-- | libcurl-agent/1.0
-- | PHP/
-- | GT::WWW
-- | Snoopy
-- | MFC_Tear_Sample
-- | HTTP::Lite
-- | PHPCrawl
-- | URI::Fetch
-- | Zend_Http_Client
-- | http client
-- | PECL::HTTP
-- | WWW-Mechanize/1.34
-- | Change in Status Code:
-- | Python-urllib/2.5: 403
-- |_ Wget/1.13.4 (linux-gnu): 403
--
-- @xmloutput
-- <elem key="Status for browser useragent">200</elem>
-- <elem key="Redirected To">https://www.example.com/</elem>
-- <table key="Allowed User Agents">
-- <elem>Mozilla/5.0 (compatible; Nmap Scripting Engine;
-- https://nmap.org/book/nse.html)</elem>
-- <elem>libwww</elem>
-- <elem>lwp-trivial</elem>
-- <elem>libcurl-agent/1.0</elem>
-- <elem>PHP/</elem>
-- <elem>GT::WWW</elem>
-- <elem>Snoopy</elem>
-- <elem>MFC_Tear_Sample</elem>
-- <elem>HTTP::Lite</elem>
-- <elem>PHPCrawl</elem>
-- <elem>URI::Fetch</elem>
-- <elem>Zend_Http_Client</elem>
-- <elem>http client</elem>
-- <elem>PECL::HTTP</elem>
-- <elem>WWW-Mechanize/1.34</elem>
-- </table>
-- <table key="Change in Status Code">
-- <elem key="Python-urllib/2.5">403</elem>
-- <elem key="Wget/1.13.4 (linux-gnu)">403</elem>
-- </table>
---
categories = {"discovery", "safe"}
author = "George Chatzisofroniou"
license = "Same as Nmap--See https://nmap.org/book/man-legal.html"
local http = require "http"
local target = require "target"
local httpspider = require "httpspider"
local shortport = require "shortport"
local stdnse = require "stdnse"
local table = require "table"
local url = require "url"
getLastLoc = function(host, port, useragent)
local options
options = {header={}, no_cache=true, bypass_cache=true, redirect_ok=function(host,port)
local c = 3
return function(url)
if ( c==0 ) then return false end
c = c - 1
return true
end
end }
options['header']['User-Agent'] = useragent
stdnse.debug2("Making a request with User-Agent: " .. useragent)
local response = http.get(host, port, '/', options)
if response.location then
return response.location[#response.location],response.status or false, response.status
end
return false, response.status
end
portrule = shortport.port_or_service( {80, 443}, {"http", "https"}, "tcp", "open")
action = function(host, port)
local moreagents = stdnse.get_script_args("http-useragent-tester.useragents") or nil
local newtargets = stdnse.get_script_args("newtargets") or nil
local output = stdnse.output_table()
-- We don't crawl any site. We initialize a crawler to use its iswithinhost method.
local crawler = httpspider.Crawler:new(host, port, '/', { scriptname = SCRIPT_NAME } )
local HTTPlibs = {
http.USER_AGENT,
"libwww",
"lwp-trivial",
"libcurl-agent/1.0",
"PHP/",
"Python-urllib/2.5",
"GT::WWW",
"Snoopy",
"MFC_Tear_Sample",
"HTTP::Lite",
"PHPCrawl",
"URI::Fetch",
"Zend_Http_Client",
"http client",
"PECL::HTTP",
"Wget/1.13.4 (linux-gnu)",
"WWW-Mechanize/1.34"
}
if moreagents then
for _, l in ipairs(moreagents) do
table.insert(HTTPlibs, l)
end
end
-- We perform a normal browser request and get the returned location
local loc, status = getLastLoc(host, port, "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17")
output['Status for browser useragent'] = status
if loc then
output['Redirected To'] = loc
end
local allowed, forb, status_changed = {}, {}, {}
for _, l in ipairs(HTTPlibs) do
local libloc, libstatus = getLastLoc(host, port, l)
-- If the library's request returned a different location, that means the request was redirected somewhere else, hence is forbidden.
if libloc and loc ~= libloc then
forb[l] = {}
local libhost = url.parse(libloc)
if not crawler:iswithinhost(libhost.host) then
forb[l]['Different Host'] = tostring(libloc)
if newtargets then
target.add(libhost.host)
end
else
forb[l]['Same Host'] = tostring(libloc)
end
elseif status ~= libstatus then
status_changed[l] = libstatus
else
table.insert(allowed, l)
end
end
if next(allowed) ~= nil then
output['Allowed User Agents'] = allowed
end
if next(forb) ~= nil then
output['Forbidden/Redirected User Agents'] = forb
end
if next(status_changed) ~= nil then
output['Change in Status Code'] = status_changed
end
return output
end