fix

crawlers.txt

@@ -1,129 +1,134 @@
-facebookexternalhit
+impendoom-bot
-YandexBot
+msnbot-media
-PetalBot
+LinkPreview
-GPTBot
+SerendeputyBot
-Amazonbot
+Arquivo-web-crawler
+Expanse
+x22Xpanse-bot
+KixxActivityPubCrawler
+Gabanzabot
+hstspreload-bot
+YisouSpider
+gotosocial
+FAST-WebCrawler
+Facebot
+Go-http-client
+MisskeyBot
+search-engine-indexer
+PixelFedBot
+lemmy-stats-crawler
+Discovery
+Twingly
+Friendica
+Podverse
+WellKnownBot
+TelegramBot
+Crawler
 SemrushBot
+bingbot
+CyberFindCrawler
+AportCatalogRobot
+LivelapBot
+duckduckbot
+YandexBot
+intelx.io_bot
+FediIndex
+Sogou
+YandexImageResizer
+Slack-ImgProxy
+ISSCyberRiskCrawler
+Mitra
+YandexRenderResourcesBot
+FriendlyCrawler
+MbinBot
+YandexImages
+Exabot
+SemanticScholarBot
+Twitterbot
+SeznamBot
+oii-research
+Horrid
+Ai2Bot-Dolma
+ZoominfoBot
+CCBot
+serpstatbot
+YandexUserproxy
+SeoCherryBot
+Amazonbot
+DotBot
+VirusTotalBot
+AwarioBot
+wpbot
+ws-bot-v1
 AhrefsBot
-slurp
+ldspider
-openai
+Googlebot-Image
-ahrefsbot
+bot
+ImagesiftBot
+Bytespider
+BW/1.2
+AwarioSmartBot
+vmcrawl
+GenomeCrawlerd
+Chodes
+facebook
+Barkrowler
+FediDB
+ev-crawler
+Everyone
+FediFetcher
+CDSCbot
+PerplexityBot
+BitSightBot
+facebookexternalhit
+DataForSeoBot
+baidu
+RedekenBot
+coccocbot-web
+GNUsocialBot
+PagePeeker
+bots.retroverse.social
+2ip
+CensysInspect
+BLEXBot
+Googlebot
+archive.org_bot
 majestic
 applebot
-duckduckbot
-baidu
-lemmy
-FediDB
-facebook
-bingbot
-MJ12bot
-lemmy-stats-crawler
-Bytespider
-CDSCbot
-Googlebot
-FediDB
-ClaudeBot
-Podverse
-Expanse
-oii-research
-DotBot
-ZoominfoBot
-LivelapBot
-rss-is-dead.lol
-FreshRSS
-Barkrowler
-DataForSeoBot
-PixelFedBot
-SerendeputyBot
-2ip bot
-GNUsocialBot
-BacklinksExtendedBot
-ws-bot-v1
-ImagesiftBot
-WellKnownBot
-FediIndex
-FriendlyCrawler
-gotosocial
-Synapse
-impendoom-bot
-Mitra
-kbinBot
-BitSightBot
-FediFetcher
-MbinBot
-Discordbot
-YisouSpider
-LinkPreview
-SurdotlyBot
-AwarioSmartBot
-msnbot-media
-msnbot
-ev-crawler
-BLEXBot
-YandexImages
-Chodes
-hstspreload-bot
-Twitterbot
-TelegramBot
-Slack-ImgProxy
-GenomeCrawlerd
-search-engine-indexer
-SemanticScholarBot
-yacybot
-BW/1.2
-Twingly
-IonCrawl
-vmcrawl
-SeoCherryBot
-coccocbot-web
-FAST-WebCrawler
-YandexImageResizer
-serpstatbot
-YandexRenderResourcesBot
-CCBot
-wpbot
-LinkedInBot
-SeznamBot
 Mail.RU_Bot
-RedekenBot
-SiteCheckerBotCrawler
-AwarioBot
-intelx.io_bot
-YandexWebmaster
-Qwant
-PagePeeker
-Sogou Push
 KOCMOHABT
-ldspider
+openai
-robots.txt
+Discordbot
-bots.retroverse.social
+lemmy
-archive.org_bot
-Facebot
-Exabot
-MisskeyBot
-ISSCyberRiskCrawler
-AportCatalogRobot
-RSS Discovery Engine
-AdsBot-Google
-CyberFind Crawler
-t3versionsBot
-CyberFindCrawler
-KixxActivityPubCrawler
-x22Xpanse-bot
-Arquivo-web-crawler
-YandexUserproxy
-Ai2Bot-Dolma
-PerplexityBot
-VirusTotalBot
-Gabanzabot
-Horrid Chodes For Everyone
 TurnitinBot
-YandexFavicon
+BacklinksExtendedBot
-Trident
 meta-externalagent
-Barkrowler
+ahrefsbot
-Googlebot-Image
+Synapse
-CensysInspect
+PetalBot
-Go-http-client
+kbinBot
-Friendica
+robots.txt
+IonCrawl
+SiteCheckerBotCrawler
+RSS
+yacybot
+FreshRSS
+YandexWebmaster
+LinkedInBot
 HeadlessChrome
+Push
+t3versionsBot
+ClaudeBot
+For
+Qwant
+msnbot
+Trident
+rss-is-dead.lol
+SurdotlyBot
+MJ12bot
+YandexFavicon
+Engine
+AdsBot-Google
+GPTBot
+CyberFind
+slurp

firewall.sh

@@ -471,8 +471,27 @@ menu() {
 	menu
 }
 
+importDB(){
+	DATA=($(cat safe.txt));for i in "${DATA[@]}"; do redis-cli SADD safe_traffic $i;done
+	DATA=($(cat bots.txt));for i in "${DATA[@]}"; do redis-cli SADD bots $i;done
+	DATA=($(cat crawlers.txt));for i in "${DATA[@]}"; do redis-cli SADD crawlers $i;done
+}
+
+exportDB(){
+	rm -f crawlers.txt
+	rm -f bots.txt
+	rm -f safe.txt
+	for i in "${CRAWLER_DB[@]}"; do echo $i >> crawlers.txt;done
+	for i in "${SAVED_BOTS[@]}"; do echo $i >> bots.txt;done
+	for i in "${SAFE_TRAFFIC[@]}"; do echo $i >> safe.txt;done
+}
+
 if [ "$1" = "start" ]; then
 	start
+elif [ "$1" = "import-db" ]; then
+	importDB
+elif [ "$1" = "export-db" ]; then
+	exportDB
 elif [ "$1" = "virt" ]; then
 	virtualization
 elif [ "$1" = "bot-search" ]; then

safe.txt

@@ -1,5 +1,6 @@
 192.168.0
-POST /inbox
+POST
+/inbox
 api/v1/instance
 api/v1/notifications
 api/v1/accounts
@@ -66,7 +67,6 @@ Mitra
 Misskey
 Lalilulelo
 Deno
-notifications
 frontend_settings
 announcements
 Husky