massive.ua robots.txt

*

Rule	Path
Disallow	/search/
Disallow	/Code%3D*
Disallow	/workbasket
Disallow	/basket/
Disallow	/login/
Disallow	*/articalbrandurl/
Disallow	/ajax.php
Disallow	/tel%3A*/
Disallow	/?utm_source
Disallow	*/image/layout/star-on.png/
Disallow	/ajax.php
Disallow	/expert/
Disallow	/mixed/
Disallow	/account/
Disallow	/login/

Rule

Path

Disallow

/search/

Disallow

/Code%3D*

Disallow

/workbasket

Disallow

/basket/

Disallow

/login/

Disallow

*/articalbrandurl/

Disallow

/ajax.php

Disallow

/tel%3A*/

Disallow

/?utm_source

Disallow

*/image/layout/star-on.png/

Disallow

/ajax.php

Disallow

/expert/

Disallow

/mixed/

Disallow

/account/

Disallow

/login/

googlebot

Rule	Path
Allow	/*.css
Allow	/*.jpg
Allow	/*.gif
Allow	/*.png
Disallow	/*?utm_
Disallow	/?gclid=
Disallow	/tel%3A
Disallow	/expert/
Disallow	/mixed/
Disallow	/account/
Disallow	/login/

Rule

Path

Allow

/*.css

Allow

/*.jpg

Allow

/*.gif

Allow

/*.png

Disallow

/*?utm_

Disallow

/*?gclid=*

Disallow

/tel%3A

Disallow

/expert/

Disallow

/mixed/

Disallow

/account/

Disallow

/login/

*

Rule	Path
Disallow	/*?utm_
Disallow	/?gclid=

Rule

Path

Disallow

/*?utm_

Disallow

/*?gclid=*

yandex

Rule	Path
Disallow	/*?utm_
Disallow	/?gclid=
Disallow	/expert/
Disallow	/mixed/
Disallow	/account/
Disallow	/login/

Rule

Path

Disallow

/*?utm_

Disallow

/*?gclid=*

Disallow

/expert/

Disallow

/mixed/

Disallow

/account/

Disallow

/login/

blexbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

yahoo

Rule	Path
Disallow	/

Rule

Path

Disallow

/

mj12bot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

ahrefsbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

baiduspider

Rule	Path
Disallow	/

Rule

Path

Disallow

/

blexbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

emailcollector

Rule	Path
Disallow	/

Rule

Path

Disallow

/

emailsiphon

Rule	Path
Disallow	/

Rule

Path

Disallow

/

bingbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

addthis

Rule	Path
Disallow	/

Rule

Path

Disallow

/

ahrefsbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

alexabot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

amznkassocbot/4.0

Rule	Path
Disallow	/

Rule

Path

Disallow

/

baidu

Rule	Path
Disallow	/

Rule

Path

Disallow

/

baiduspider

Rule	Path
Disallow	/

Rule

Path

Disallow

/

baiduspider-image

Rule	Path
Disallow	/

Rule

Path

Disallow

/

bdcbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

bdcbot/1.0

Rule	Path
Disallow	/

Rule

Path

Disallow

/

blexbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

bsalsa

Rule	Path
Disallow	/

Rule

Path

Disallow

/

butterfly

Rule	Path
Disallow	/

Rule

Path

Disallow

/

ccbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

changedetection

Rule	Path
Disallow	/

Rule

Path

Disallow

/

coccoc

Rule	Path
Disallow	/

Rule

Path

Disallow

/

crazywebcrawler-spider

Rule	Path
Disallow	/

Rule

Path

Disallow

/

daumoa

Rule	Path
Disallow	/

Rule

Path

Disallow

/

docomo/2.0

Rule	Path
Disallow	/

Rule

Path

Disallow

/

dotbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

easouspider

Rule	Path
Disallow	/

Rule

Path

Disallow

/

exabot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

ezooms

Rule	Path
Disallow	/

Rule

Path

Disallow

/

findxbot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

findxbot/1.0

Rule	Path
Disallow	/

Rule

Path

Disallow

/

flipboard

Rule	Path
Disallow	/

Rule

Path

Disallow

/

genieo

Rule	Path
Disallow	/

Rule

Path

Disallow

/

gluten free crawler

Rule	Path
Disallow	/

Rule

Path

Disallow

/

grapeshot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

haosouspider

Rule	Path
Disallow	/

Rule

Path

Disallow

/

hypercrawl

Rule	Path
Disallow	/

Rule

Path

Disallow

/

ichiro

Rule	Path
Disallow	/

Rule

Path

Disallow

/

ichiro/3.0

Rule	Path
Disallow	/

Rule

Path

Disallow

/

ichiro/4.0

Rule	Path
Disallow	/

Rule

Path

Disallow

/

infoweb

Rule	Path
Disallow	/

Rule

Path

Disallow

/

istellabot

Rule	Path
Disallow	/

Rule

Path

Disallow

/

libwww-perl

Rule

Path

Disallow

/

linguee bot

Rule

Path

Disallow

/

linkdexbot

Rule

Path

Disallow

/

linkdexbot/2.0

Rule

Path

Disallow

/

linkdexbot/2.1

Rule

Path

Disallow

/

linkdexbot/2.2

Rule

Path

Disallow

/

linkpadbot

Rule

Path

Disallow

/

ltx71

Rule

Path

Disallow

/

meanpathbot

Rule

Path

Disallow

/

megaindex.ru

Rule

Path

Disallow

/

megaindex.ru/2.0

Rule

Path

Disallow

/

naverbot

Rule

Path

Disallow

/

nerdybot

Rule

Path

Disallow

/

netseer

Rule

Path

Disallow

/

nutch bots

Rule

Path

Disallow

/

obot

Rule

Path

Disallow

/

paperlibot

Rule

Path

Disallow

/

phpservermon/3.1.1

Rule

Path

Disallow

/

privacyawarebot/1.1

Rule

Path

Disallow

/

proximic

Rule

Path

Disallow

/

psbot

Rule

Path

Disallow

/

queryseekerspider

Rule

Path

Disallow

/

qwantify

Rule

Path

Disallow

/

rogerbot

Rule

Path

Disallow

/

screenerbot

Rule

Path

Disallow

/

searchmetricsbot

Rule

Path

Disallow

/

sf/2 (skipfish)

Rule

Path

Disallow

/

siteexplorer

Rule

Path

Disallow

/

sogou head spider/3.0

Rule

Path

Disallow

/

sogou orion spider/3.0

Rule

Path

Disallow

/

sogou pic agent

Rule

Path

Disallow

/

sogou pic spider/3.0

Rule

Path

Disallow

/

sputnikbot

Rule

Path

Disallow

/

sunrise

Rule

Path

Disallow

/

surdotlybot

Rule

Path

Disallow

/

surdotlybot

Rule

Path

Disallow

/

swiftbot

Rule

Path

Disallow

/

tdjbot

Rule

Path

Disallow

/

trovitbot

Rule

Path

Disallow

/

turnitinbot

Rule

Path

Disallow

/

tweetmeme

Rule

Path

Disallow

/

tweetmemebot

Rule

Path

Disallow

/

twengabot

Rule

Path

Disallow

/

twengabot-2.0

Rule

Path

Disallow

/

twengabot-discover

Rule

Path

Disallow

/

twengabot/2.0

Rule

Path

Disallow

/

unwindfetchor

Rule

Path

Disallow

/

uptimebot

Rule

Path

Disallow

/

uptimebot/1.0

Rule

Path

Disallow

/

uptimerobot

Rule

Path

Disallow

/

uptimerobot/2.0

Rule

Path

Disallow

/

voltron

Rule

Path

Disallow

/

wbsearchbot

Rule

Path

Disallow

/

wesee

Rule

Path

Disallow

/

wget

Rule

Path

Disallow

/

wget

Rule

Path

Disallow

/

wotbox

Rule

Path

Disallow

/

xovibot

Rule

Path

Disallow

/

y! j-asr

Rule

Path

Disallow

/

yacybot

Rule

Path

Disallow

/

yeti

Rule

Path

Disallow

/

yodaobot

Rule

Path

Disallow

/

yoozbot-2.2

Rule

Path

Disallow

/

youdaobot/1.0

Rule

Path

Disallow

/

yahoo! slurp

Rule

Path

Disallow

/

yahoo-slurp

Rule

Path

Disallow

/

anews/1.0

Rule

Path

Disallow

/

anews

Rule

Path

Disallow

/

kribrum

Rule

Path

Disallow

/

kribrum/0.1

Rule

Path

Disallow

/

iskanie

Rule

Path

Disallow

/

paperlibot/2.1

Rule

Path

Disallow

/

special_archiver/3.1.1

Rule

Path

Disallow

/

special_archiver

Rule

Path

Disallow

/

ccbot/2.0

Rule

Path

Disallow

/

exabot/3.0

Rule

Path

Disallow

/

um-fc

Rule

Path

Disallow

/

um-fc/1.0

Rule

Path

Disallow

/

um-ln

Rule

Path

Disallow

/

um-ln/1.0

Rule

Path

Disallow

/

heritrix

Rule

Path

Disallow

/

deusu/5.0.2

Rule

Path

Disallow

/

bitlybot

Rule

Path

Disallow

/

bitlybot/3.0

Rule

Path

Disallow

/

maxpointcrawler

Rule

Path

Disallow

/

getintent crawler

Rule

Path

Disallow

/

scrapy

Rule

Path

Disallow

/

scrapy/1.2.2

Rule

Path

Disallow

/

twitterbot

Rule

Path

Disallow

/

twitterbot/1.0

Rule

Path

Disallow

/

sputnikbot/2.3

Rule

Path

Disallow

/

trident/6.0

Rule

Path

Disallow

/

trident/5.0

Rule

Path

Disallow

/

trident

Rule

Path

Disallow

/

guzzlehttp/6.1.0

Rule

Path

Disallow

/

guzzlehttp

Rule

Path

Disallow

/

x11

Rule

Path

Disallow

/

msnbot

Rule

Path

Disallow

/

bingbot

Rule

Path

Disallow

/

bingbot/2.0

Rule

Path

Disallow

/

stackrambler

Rule

Path

Disallow

/

metaspider

Rule

Path

Disallow

/

aport

Rule

Path

Disallow

/

scooter

Rule

Path

Disallow

/

trendictionbot

Rule

Path

Disallow

/

trendictionbot0.5.0

Rule

Path

Disallow

/

kraken

Rule

Path

Disallow

/

kraken/0.1

Rule

Path

Disallow

/

dotbot/1.1

Rule

Path

Disallow

/

bubing

Rule

Path

Disallow

/

yeti/1.1

Rule

Path

Disallow

/

seznambot/3.2

Rule

Path

Disallow

/

webartexbot

Rule

Path

Disallow

/

blexbot/1.0

Rule

Path

Disallow

/

spbot/5.0.3

Rule

Path

Disallow

/

spbot

Rule

Path

Disallow

/

um-ic

Rule

Path

Disallow

/

um-ic/1.0

Rule

Path

Disallow

/

germcrawler

Rule

Path

Disallow

/

sogou

Rule

Path

Disallow

/

gingercrawler

Rule

Path

Disallow

/

nutch

Rule

Path

Disallow

/

urlbot

Rule

Path

Disallow

/

scoutjet

Rule

Path

Disallow

/

grouphigh

Rule

Path

Disallow

/

grouphigh/1.0

Rule

Path

Disallow

/

mj12bot/v1.4.3

Rule

Path

Disallow

/

mj12bot/v1.4.0

Rule

Path

Disallow

/

mj12bot/v1.2.4

Rule

Path

Disallow

/

mj12bot/v1.2.3

Rule

Path

Disallow

/

mj12bot/v1.0.8

Rule

Path

Disallow

/

mj12bot/v1.0.7

Rule

Path

Disallow

/

mj12bot/v1.0.6

Rule

Path

Disallow

/

mj12bot/v1.0.5

Rule

Path

Disallow

/

mj12bot/v1.4.5

Rule

Path

Disallow

/

mj12bot/v1.4.6

Rule

Path

Disallow

/

mj12bot/v1.4.7

Rule

Path

Disallow

/

mj12bot/v1.4.8

Rule

Path

Disallow

/

rogerbot/1.0

Rule

Path

Disallow

/

rogerbot

Rule

Path

Disallow

/

megaindex

Rule

Path

Disallow

/

megaindex.ru

Rule

Path

Disallow

/

megaindex

Rule

Path

Disallow

/

crawler4j

Rule

Path

Disallow

/

steeler/3.5

Rule

Path

Disallow

/

steeler

Rule

Path

Disallow

/

ahrefsbot/5.1

Rule

Path

Disallow

/

ahrefsbot/5.2

Rule

Path

Disallow

/

ahrefsbot/1.0

Rule

Path

Disallow

/

ahrefsbot/2.0

Rule

Path

Disallow

/

ahrefsbot/3.0

Rule

Path

Disallow

/

ahrefsbot/4.0

Rule

Path

Disallow

/

ahrefsbot/5.0

Rule

Path

Disallow

/

ahrefsbot/3.1

Rule

Path

Disallow

/

serpstatbot

Rule

Path

Disallow

/

boitho

Rule

Path

Disallow

/

ia_archiver

Rule

Path

Disallow

/

baiduspider

Rule

Path

Disallow

/

sogou spider

Rule

Path

Disallow

/

yodaobot

Rule

Path

Disallow

/

sputnikbot

Rule

Path

Disallow

/

moget

Rule

Path

Disallow

/

ichiro

Rule

Path

Disallow

/

naverbot

Rule

Path

Disallow

/

yeti

Rule

Path

Disallow

/

obot

Rule

Path

Disallow

/

addthis

Rule

Path

Disallow

/

wotbox

Rule

Path

Disallow

/

embedly

Rule

Path

Disallow

/

paperlibot

Rule

Path

Disallow

/

genieo

Rule

Path

Disallow

/

showyoubot

Rule

Path

Disallow

/

tweetmemebot

Rule

Path

Disallow

/

ahrefsbot

Rule

Path

Disallow

/

exabot

Rule

Path

Disallow

/

surveybot

Rule

Path

Disallow

/

sistrix

Rule

Path

Disallow

/

mj12bot

Rule

Path

Disallow

/

bdcbot

Rule

Path

Disallow

/

bdcbot/1.0

Rule

Path

Disallow

/

rogerbot

Rule

Path

Disallow

/

spbot

Rule

Path

Disallow

/

linkdexbot

Rule

Path

Disallow

/

ccbot

Rule

Path

Disallow

/

haosouspider

Rule

Path

Disallow

/

dotbot

Rule

Path

Disallow

/

grapeshot

Rule

Path

Disallow

/

proximic

Rule

Path

Disallow

/

linkpadbot

Rule

Path

Disallow

/

sunrise

Rule

Path

Disallow

/

butterfly

Rule

Path

Disallow

/

linguee bot

Rule

Path

Disallow

/

twengabot-2.0

Rule

Path

Disallow

/

turnitinbot

Rule

Path

Disallow

/

blexbot

Rule

Path

Disallow

/

amznkassocbot/4.0

Rule

Path

Disallow

/

nerdybot

Rule

Path

Disallow

/

trovitbot

Rule

Path

Disallow

/

shopwiki

Rule

Path

Disallow

/

siteexplorer

Rule

Path

Disallow

/

xovibot

Rule

Path

Disallow

/

smtbot

Rule

Path

Disallow

/

meanpathbot

Rule

Path

Disallow

/

mixrankbot

Rule

Path

Disallow

/

easouspider

Rule

Path

Disallow

/

riddler

Rule

Path

Disallow

/

seznambot

Rule

Path

Disallow

/

swiftbot

Rule

Path

Disallow

/

qwantify

Rule

Path

Disallow

/

psbot

Rule

Path

Disallow

/

crazywebcrawler-spider

Rule

Path

Disallow

/

hypercrawl

Rule

Path

Disallow

/

daumoa

Rule

Path

Disallow

/

coccoc

Rule

Path

Disallow

/

netseer

Rule

Path

Disallow

/

backlinkcrawler

Rule

Path

Disallow

/

alexabot

Rule

Path

Disallow

/

surdotlybot

Rule

Path

Disallow

/

megaindex.ru/2.0

Rule

Path

Disallow

/

megaindex.ru

Rule

Path

Disallow

/

yacybot

Rule

Path

Disallow

/

queryseekerspider

Rule

Path

Disallow

/

wbsearchbot

Rule

Path

Disallow

/

deusu

Rule

Path

Disallow

/

tdjbot

Rule

Path

Disallow

/

findxbot/1.0

Rule

Path

Disallow

/

findxbot

Rule

Path

Disallow

/

y! j-asr

Rule

Path

Disallow

/

infoweb

Rule

Path

Disallow

/

nutch bots

Rule

Path

Disallow

/

python-urllib

Rule

Path

Disallow

/

ezooms

Rule

Path

Disallow

/

unwindfetchor

Rule

Path

Disallow

/

flipboard

Rule

Path

Disallow

/

voltron

Rule

Path

Disallow

/

istellabot

Rule

Path

Disallow

/

silk

Rule

Path

Disallow

/

wget

Rule

Path

Disallow

/

uptimerobot

Rule

Path

Disallow

/

uptimerobot/2.0

Rule

Path

Disallow

/

screenerbot

Rule

Path

Disallow

/

searchmetricsbot

Rule

Path

Disallow

/

wesee

Rule

Path

Disallow

/

linkdexbot/2.0

Rule

Path

Disallow

/

linkdexbot/2.1

Rule

Path

Disallow

/

linkdexbot/2.2

Rule

Path

Disallow

/

uptimebot

Rule

Path

Disallow

/

uptimebot/1.0

Rule

Path

Disallow

/

lipperhey

Rule

Path

Disallow

/

ltx71

Rule

Path

Disallow

/

python-urllib/2.7

Rule

Path

Disallow

/

sogou web spider/4.0

Rule

Path

Disallow

/

baiduspider/2.0

Rule

Path

Disallow

/

privacyawarebot/1.1

Rule

Path

Disallow

/

yoozbot-2.2

Rule

Path

Disallow

/

wget

Rule

Path

Disallow

/

tweetmeme

Rule

Path

Disallow

/

docomo/2.0

Rule

Path

Disallow

/

ichiro/4.0

Rule

Path

Disallow

/

ichiro/3.0

Rule

Path

Disallow

/

twengabot-discover

Rule

Path

Disallow

/

twengabot/2.0

Rule

Path

Disallow

/

twengabot

Rule

Path

Disallow

/

bendercrawler

Rule

Path

Disallow

/

baidu

Rule

Path

Disallow

/

bsalsa

Rule

Path

Disallow

/

phpservermon/3.1.1

Rule

Path

Disallow

/

gluten free crawler/1.0

Rule

Path

Disallow

/

gluten free crawler

Rule

Path

Disallow

/

sogou pic spider/3.0

Rule

Path

Disallow

/

sogou head spider/3.0

Rule

Path

Disallow

/

sogou orion spider/3.0

Rule

Path

Disallow

/

sogou-test-spider/4.0

Rule

Path

Disallow

/

sogou pic agent

Rule

Path

Disallow

/

baiduspider+

Rule

Path

Disallow

/

baiduspider-video

Rule

Path

Disallow

/

baiduspider-image

Rule

Path

Disallow

/

youdaobot/1.0

Rule

Path

Disallow

/

changedetection

Rule

Path

Disallow

/

surdotlybot

Rule

Path

Disallow

/

libwww-perl

Rule

Path

Disallow

/

mojeekbot

Rule

Path

Disallow

/

sogou web spider

Rule

Path

Disallow

/

Site Domain	massive.ua
Base Domain	massive.ua
Scan Status	Failed
Failure Stage	Fetching resource.
Failure Reason	Server returned a client error.
Last Scan	2024-09-02T07:03:49+00:00
Next Scan	2024-12-01T07:03:49+00:00

Scanned	2024-01-14T07:01:47+00:00
URL	https://massive.ua/robots.txt
Domain IPs	46.4.102.221
Response IP	46.4.102.221
Found	Yes
Hash	101b1e6ec2455ee2f38e3b41fe6b6639e58b09a9dd0833fb915a590c92234f75
SimHash	7074434aec33

massive.uarobots.txt

Resource Scan

Scan Details

Last Successful Scan

Groups

*

googlebot

*

yandex

blexbot

yahoo

mj12bot

ahrefsbot

baiduspider

blexbot

emailcollector

emailsiphon

bingbot

addthis

ahrefsbot

alexabot

amznkassocbot/4.0

baidu

baiduspider

baiduspider-image

bdcbot

bdcbot/1.0

blexbot

bsalsa

butterfly

ccbot

changedetection

coccoc

crazywebcrawler-spider

daumoa

docomo/2.0

dotbot

easouspider

exabot

ezooms

findxbot

findxbot/1.0

flipboard

genieo

gluten free crawler

grapeshot

haosouspider

hypercrawl

ichiro

ichiro/3.0

ichiro/4.0

infoweb

istellabot

libwww-perl

linguee bot

linkdexbot

linkdexbot/2.0

linkdexbot/2.1

linkdexbot/2.2

linkpadbot

ltx71

meanpathbot

megaindex.ru

megaindex.ru/2.0

naverbot

nerdybot

netseer

nutch bots

obot

paperlibot

phpservermon/3.1.1

privacyawarebot/1.1

proximic

psbot

queryseekerspider

qwantify

rogerbot

screenerbot

searchmetricsbot

sf/2 (skipfish)

massive.ua
robots.txt