timesofassam.com
robots.txt

Robots Exclusion Standard data for timesofassam.com

Resource Scan

Scan Details

Site Domain timesofassam.com
Base Domain timesofassam.com
Scan Status Ok
Last Scan2024-11-12T10:33:28+00:00
Next Scan 2024-11-19T10:33:28+00:00

Last Scan

Scanned2024-11-12T10:33:28+00:00
URL https://timesofassam.com/robots.txt
Domain IPs 162.214.80.64
Response IP 162.214.80.64
Found Yes
Hash 7a02e6f2e26bc7a9eb358a51a6100a77303d2b27cbebf884a1c6a9c88617f9fe
SimHash 4051c290573b

Groups

ia_archiver

Rule Path
Disallow /

archive.org_bot

Rule Path
Disallow /

ia_archiver-web.archive.org

Rule Path
Disallow /

applebot

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

baiduspider

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

bingbot

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

discordbot

Rule Path
Disallow /
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

facebookexternalhit

Rule Path
Disallow /
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

googlebot

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

googlebot-image

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /map_tile.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /static_map.php
Disallow /tr/
Disallow /tr?

ia_archiver

Rule Path
Disallow /
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

linkedinbot

Rule Path
Disallow /
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

msnbot

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

naverbot

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

pinterestbot

Rule Path
Disallow /
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

screaming frog seo spider

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

seznambot

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

slurp

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

teoma

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

telegrambot

Rule Path
Disallow /
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

twitterbot

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

yandex

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

yeti

Rule Path
Disallow /ajax/
Disallow /album.php
Disallow /checkpoint/
Disallow /contact_importer/
Disallow /dialog/
Disallow /fbml/ajax/dialog/
Disallow /feeds/
Disallow /file_download.php
Disallow /job_application/
Disallow /l.php
Disallow /moments_app/
Disallow /p.php
Disallow /photo.php
Disallow /photos.php
Disallow /plugins/
Disallow /share.php
Disallow /share/
Disallow /sharer.php
Disallow /sharer/
Disallow /tr/
Disallow /tr?

applebot

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

baiduspider

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

bingbot

Rule Path
Allow /*/videos/
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/
Allow /watch

discordbot

Rule Path
Allow /*/videos/
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/
Allow /watch/?v=*

facebookexternalhit

Rule Path
Allow /*/videos/
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/
Allow /watch/?v=*

googlebot

Rule Path
Allow /*/videos/
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/
Allow /watch

ia_archiver

Rule Path
Allow /about/privacy
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /full_data_use_policy
Allow /legal/terms
Allow /policy.php
Allow /safetycheck/

linkedinbot

Rule Path
Allow /*/videos/
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/
Allow /watch/?v=*

msnbot

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

naverbot

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

pinterestbot

Rule Path
Allow /*/videos/
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/
Allow /watch/?v=*

screaming frog seo spider

Rule Path
Allow /*/videos/
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/
Allow /watch

seznambot

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

slurp

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

teoma

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

telegrambot

Rule Path
Allow /*/videos/
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/
Allow /watch/?v=*

twitterbot

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

yandex

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

yeti

Rule Path
Allow /ajax/bootloader-endpoint/
Allow /ajax/pagelet/generic.php/PagePostsSectionPagelet
Allow /careers/
Allow /safetycheck/

*

Rule Path
Disallow /

Comments

  • Test