freedirectory.ca
robots.txt

Groups

blexbot
blackwidow
nutch
jetbot
webvac
stanford
scooter
naver
dumbot
hatena\ antenna
grub
looksmart
webzip
larbin
b2w/0.1
copernic
psbot
python-urllib
netmechanic
url_spider_pro
cherrypicker
emailcollector
emailsiphon
webbandit
emailwolf
email
extractorpro
copyrightcheck
crescent
sitesnagger
prowebwalker
cheesebot
lnspiderguy
ia_archiver
alexibot
teleport
miixpc
telesoft
website\ quester
moget
webstripper
websauger
webcopier
netants
mister\ pix
webauto
thenomad
www-collector-e
rma
libweb/clshttp
asterias
httplib
turingos
spanner
harvest
infonavirobot
bullseye
webbandit
nicerspro
microsoft\ url\ control
dittospyder
foobot
webmasterworldforumbot
spankbot
botalot
lwp-trivial
webmasterworld
bunnyslippers
urly\ warning
wget
linkwalker
cosmos
hloader
humanlinks
linkextractorpro
offline\ explorer
mata\ hari
lexibot
web\ image\ collector
the\ intraformant
true_robot
blowfish
searchengineworld
jennybot
miixpc
builtbottough
propowerbot
backdoorbot
tocrawl/urldispatcher
webenhancer
suzuran
webviewer
vci
szukacz
queryn
openfind
openbot
webster
erocrawler
linkscan
keyword
kenjin
iron33
bookmark\ search\ tool
getright
fairad\ client
gaisbot
aqua_products
radiation\ retriever\ 1.1
flaming\ attackbot
oracle\ ultra\ search
msiecrawler
perman
searchpreview
sootle
enterprise_search
bot\ mailto:craftbot@yahoo.com
chinaclaw
custo
disco
download\ demon
ecatch
eirgrabber
emailsiphon
emailwolf
express\ webpictures
extractorpro
eyenetie
flashget
getright
getweb!
go!zilla
go-ahead-got-it
grabnet
grafula
hmview
httrack
image\ stripper
image\ sucker
indy\ library
interget
internet\ ninja
jetcar
joc\ web\ spider
larbin
leechftp
mass\ downloader
midown\ tool
mister\ pix
navroad
nearsite
netants
netspider
net\ vampire
netzip
octopus
offline\ explorer
offline\ navigator
pagegrabber
papa\ foto
pavuk
pcbrowser
realdownload
reget
sitesnagger
smartdownload
superbot
superhttp
surfbot
takeout
teleport\ pro
voideye
web\ image\ collector
web\ sucker
webauto
webcopier
webfetch
webgo\ is
webleacher
webreaper
websauger
website\ extractor
website\ quester
webstripper
webwhacker
webzip
wget
widow
wwwoffle
xaldon\ webspider
zeus
semrush
becomebot
ahrefsbot
mj12bot
rogerbot
exabot
xenu
dotbot
gigabot
sp_auditbot
semrushbot
semrushbot-sa
ahrefs
ahrefs.com
ahrefsbot/2.0
ahrefsbot/3.1
backlink-check.de
backlinkcrawler
baiduspider
baiduspider-image
baiduspider-video
birubot
bixolabs
botonparade
botrighthere
checkbot
cloudservermarketspider
cognitiveseo
crazywebcrawler-spider
botonparade
botrighthere
checkbot
cloudservermarketspider
cognitiveseo
crazywebcrawler-spider
domaincrawler
duggmirror
eurobot
extlinksbot
ezooms robot
fasterfox
findlinks robot
fr-crawler
huaweisymantecspider
iccrawler - icjobs
internetseer
iwebtool
jamesbot
jobs.de-robot
linguee
magpie-crawler
meanpathbot
megaindex.com
megaindex.ru
mojeekbot
monitorbacklinks
mozilla/5.0 (compatible; ravencrawler/2.0; +https
nerdbynature.bot
netestate
oneriot
openfind
openfind data gatherer
python/3.5 aiohttp
ranksignals
ruby
scoutjet
searchmetricsbot
seodiver
seokicks
seokicks-robot
seoprofiler
serpstatbot
shopwiki crawler
sistrix crawler
sogou spider
sosospider
spbot
speedy
spinn3r
szukacz/1.4
tighttwatbot
toweya.com
trendictionbot
twenga.com
twenga2.com
unisterbot
unwindfetchor
updownerbot
voilabot
wbsearchbot
wotbox
yodaobot
youdaobot
blekkobot
semrushbot
semrushbot
semrushbot-sa
semrushbot-sa
searchmetricsbot
seokicks-robot
sistrix
lipperhey spider
ncbot
backlinkcrawler
archive.org_bot
meanpathbot
pagesinventory
aboundexbot
spbot
linkdexbot
ezooms
scoutjet
dsearch
majestic-12
majestic-seo
lipperhey
lipperhey-kaus-australis
lipperhey seo service
lipperhey-kaus-australis/5.0