# Block social media crawlers from accessing user-specific content
User-agent: facebookexternalhit
Crawl-delay: 30
Request-rate: 1/30s
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /reset-password
Disallow: /verify-email

User-agent: Facebot
Crawl-delay: 30
Request-rate: 1/30s
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /reset-password
Disallow: /verify-email

User-agent: Twitterbot
Crawl-delay: 30
Request-rate: 1/30s
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /reset-password
Disallow: /verify-email

User-agent: LinkedInBot
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /reset-password
Disallow: /verify-email

User-agent: DiscordBot
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /reset-password
Disallow: /verify-email

User-agent: WhatsApp
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /reset-password
Disallow: /verify-email

User-agent: TelegramBot
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /reset-password
Disallow: /verify-email

User-agent: SlackBot
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /reset-password
Disallow: /verify-email

User-agent: SkypeUriPreview
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /reset-password
Disallow: /verify-email

User-agent: bingbot
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin

User-agent: DuckDuckBot
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin

# Block aggressive/malicious crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MajesticSEO
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: YandexBot
Crawl-delay: 10
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /account
Disallow: /admin

User-agent: Googlebot
Crawl-delay: 1
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /account
Disallow: /admin
Disallow: /api/

# Allow general search engines to index public pages with rate limiting
User-agent: *
Crawl-delay: 5
Allow: /
Disallow: /project
Disallow: /profile
Disallow: /auth
Disallow: /login
Disallow: /signup
Disallow: /account
Disallow: /admin
Disallow: /api/
Disallow: /_next/
Disallow: /reset-password
Disallow: /verify-email
Disallow: /dashboard
Disallow: /settings
Disallow: /config
Disallow: /private
Disallow: /user/
Disallow: /users/
Disallow: /member/
Disallow: /members/
Disallow: /internal/
Disallow: /secure/
Disallow: /protected/
Disallow: /tmp/
Disallow: /temp/
Disallow: /cache/
Disallow: /backup/
Disallow: /backups/
Disallow: /logs/
Disallow: /log/
Disallow: /debug/
Disallow: /test/
Disallow: /tests/
Disallow: /staging/
Disallow: /dev/
Disallow: /development/
Disallow: /.env
Disallow: /.git/
Disallow: /.svn/
Disallow: /.htaccess
Disallow: /.htpasswd
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /wp-config.php
Disallow: /cgi-bin/
Disallow: /bin/
Disallow: /etc/
Disallow: /var/
Disallow: /proc/
Disallow: /sys/
Disallow: /root/
Disallow: /home/
Disallow: /phpmyadmin/
Disallow: /phpMyAdmin/
Disallow: /mysql/
Disallow: /database/
Disallow: /db/
Disallow: *.sql$
Disallow: *.bak$
Disallow: *.backup$
Disallow: *.zip$
Disallow: *.tar$
Disallow: *.gz$
Disallow: *.log$
Disallow: *.txt$
Disallow: *.json$
Disallow: *.xml$
Disallow: *.csv$
Disallow: /error/
Disallow: /errors/
Disallow: /404/
Disallow: /500/
Disallow: /maintenance/
Disallow: /offline/

# Allow crawling of specific static assets only
Allow: /favicon.ico
Allow: /apple-touch-icon*.png
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /manifest.json
# Allow specific public image formats
Allow: /public/*.png$
Allow: /public/*.jpg$
Allow: /public/*.jpeg$
Allow: /public/*.svg$
Allow: /images/*.png$
Allow: /images/*.jpg$
Allow: /images/*.jpeg$
Allow: /images/*.svg$
# Block all other file types by default for security

Sitemap: https://credwork.co/sitemap.xml