Nginx反爬虫策略，防止UA抓取网站

  vim /usr/www/server/nginx/conf/anti_spider.conf

  #禁止Scrapy等工具的抓取   if ($http_user_agent ~* (Scrapy|Curl|HttpClient)) {      return 403;   }   #禁止指定UA及UA为空的访问   if ($http_user_agent ~ "WinHttp|WebZIP|FetchURL|node-superagent|java/|FeedDemon|Jullo|JikeSpider|Indy Library|Alexa Toolbar|AskTbFXTV|AhrefsBot|CrawlDaddy|Java|Feedly|Apache-HttpAsyncClient|UniversalFeedParser|ApacheBench|Microsoft URL Control|Swiftbot|ZmEu|oBot|jaunty|Python-urllib|lightDeckReports Bot|YYSpider|DigExt|HttpClient|MJ12bot|heritrix|EasouSpider|Ezooms|BOT/0.1|YandexBot|FlightDeckReports|Linguee Bot|^$" ) {      return 403;          }   #禁止非GET|HEAD|POST方式的抓取   if ($request_method !~ ^(GET|HEAD|POST)$) {     return 403;   }  #屏蔽单个IP的命令是  #deny 123.45.6.7  #封整个段即从123.0.0.1到123.255.255.254的命令  #deny 123.0.0.0/8  #封IP段即从123.45.0.1到123.45.255.254的命令  #deny 124.45.0.0/16  #封IP段即从123.45.6.1到123.45.6.254的命令是  #deny 123.45.6.0/24  # 以下IP皆为流氓  #deny 58.95.66.0/24;

  # 反爬虫    include /usr/www/server/nginx/conf/anti_spider.conf

  λ curl -X GET -I -A 'YYSpider' https://www.myong.top  HTTP/1.1 200 Connection established  HTTP/2 403  server: marco/2.11  date: Fri, 20 Mar 2020 08:48:50 GMT  content-type: text/html  content-length: 146  x-source: C/403  x-request-id: 3ed800d296a12ebcddc4d61c57500aa2

  λ curl -X GET -I -A 'BaiduSpider' https://www.myong.top  HTTP/1.1 200 Connection established  HTTP/2 200  server: marco/2.11  date: Fri, 20 Mar 2020 08:49:47 GMT  content-type: text/html  vary: Accept-Encoding  x-source: C/200  last-modified: Wed, 18 Mar 2020 13:16:50 GMT  etag: "5e721f42-150ce"  x-request-id: e82999a78b7d7ea2e9ff18b6f1f4cc84

  FeedDemon       内容采集   BOT/0.1 (BOT for JCE) sql注入   CrawlDaddy      sql注入   Java         内容采集   Jullo         内容采集   Feedly        内容采集   UniversalFeedParser  内容采集   ApacheBench      cc攻击器   Swiftbot       无用爬虫   YandexBot       无用爬虫   AhrefsBot       无用爬虫   YisouSpider      无用爬虫（已被UC神马搜索收购，此蜘蛛可以放开！）   jikeSpider      无用爬虫   MJ12bot        无用爬虫   ZmEu phpmyadmin    漏洞扫描   WinHttp        采集cc攻击   EasouSpider      无用爬虫   HttpClient      tcp攻击   Microsoft URL Control 扫描   YYSpider       无用爬虫   jaunty        wordpress爆破扫描器   oBot         无用爬虫   Python-urllib     内容采集   Indy Library     扫描   FlightDeckReports Bot 无用爬虫   Linguee Bot      无用爬虫

您可能感兴趣的文章: