bug fix: DNS response too slow

README.md

@@ -1,4 +1,4 @@
-# eyes.sh 1.0.2
+# eyes.sh 1.0.3
 
 View this doc in [English](README_EN.md).
 
@@ -11,6 +11,7 @@ View this doc in [English](README_EN.md).
 
 ## Chang Log 
 
+* 2024-06-02： 修复在DNS查询中串联IP查询功能，高并发请求时，响应慢的问题
 * 2022-04-09： 增加多语言支持，添加英文翻译
 * 2022-04-06： 注册用户名，限制只允许使用字母数字
 

README_EN.md

@@ -1,4 +1,4 @@
-# eyes.sh 1.0.2
+# eyes.sh 1.0.3
 
 View this doc in [Chinese](README.md).
 
@@ -19,6 +19,7 @@ Demo site  [http://eyes.sh/](http://eyes.sh/)
 
 ## Chang Log 
 
+* 2024-06-02： Bug fixed, a sync IP to geo location search (by HTTP) made DNS Server response too slow, especially under high concurrency queries
 * 2022-04-09： Add english translation
 
 Install

dnslog/urls.py

@@ -22,6 +22,7 @@ urlpatterns = [
     re_path(r'^rebind/$', views.rebind_view, name='rebind_view'),
     re_path(r'^rebind/gen$', views.rebind_gen, name='rebind_gen'),
     re_path(r'^as_admin/$', views.as_admin, name='as_admin'),
+    re_path(r'^ip2location$', views.ip_to_location, name='ip_to_location'),
     re_path(r'^config/update$', views.config_update, name='config_update'),
     path('i18n/', include('django.conf.urls.i18n')),
     re_path(r'^.*$', views.index, name='index'),

logview/models.py

@@ -54,6 +54,7 @@ class DNSLog(models.Model):
     id = models.AutoField(primary_key=True)
     user = models.ForeignKey(User, on_delete=models.CASCADE)
     host = models.CharField(max_length=255, db_index=True, null=False)
+    sub_name = models.CharField(max_length=255, db_index=True, null=True)
     type = models.CharField(max_length=8, db_index=True, null=False)
     created_time = models.DateTimeField(auto_now_add=True)
     ip = models.GenericIPAddressField(null=False, db_index=True)
@@ -64,7 +65,9 @@ class DNSLog(models.Model):
 
     class Meta:
         ordering = ['-id']
-
+        indexes = [
+            models.Index(fields=['sub_name', 'user_id']),
+        ]
 
 class DNSLogAdmin(admin.ModelAdmin):
     list_display = ('user', 'host', 'type', 'created_time')

logview/views.py

@@ -20,11 +20,14 @@ from django.contrib.auth import authenticate, login
 from django.conf import settings
 from django.contrib.auth import logout
 from django.utils.translation import gettext as _
+from django.db.models import Q
 
 
 def get_city_by_ip(ip):
     try:
-        doc = requests.get('https://whois.pconline.com.cn/ip.jsp?ip=%s' % ip).text.strip()
+        doc = requests.get('https://whois.pconline.com.cn/ip.jsp?ip=%s' % ip,
+                           headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
+                                                  '(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'}).text.strip()
         city = doc.split(' ')[0]
     except Exception as e:
         city = ''
@@ -402,12 +405,14 @@ def api(request, type, username, prefix):
 
 def group_api(request, username, prefix):
     token = request.GET.get('token', '')
-    if not User.objects.filter(username=username, token=token):
+    user = User.objects.filter(username=username, token=token)
+    if not user:
         return HttpResponse('Invalid token')
     postfix = ".%s.%s.%s" % (prefix, username, settings.DNS_DOMAIN)
-    res = DNSLog.objects.filter(host__endswith=postfix).order_by('host').values('host').distinct()
+    sub_name = prefix.split('.')[-1]
+    res = DNSLog.objects.filter(Q(sub_name=sub_name) & Q(user=user[0])).order_by('host').values('host').distinct()[:50]
     if res:
-        res = res[:50]
+        # res = res[:50]
         data = [item['host'].replace(postfix, '') for item in res]
         text = json.dumps({"success": "true", "data": data})
         return HttpResponse(text, content_type="application/json")
@@ -429,6 +434,21 @@ def as_admin(request):
     return redirect('/')
 
 
+
+@csrf_exempt
+def ip_to_location(request):
+    ips = request.POST.get('ips', '')
+    if not ips:
+        return HttpResponse('')
+    matches = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ips)
+    found = {}
+    if matches:
+        for ip in matches[:10]:
+            city = get_city_by_ip(ip)
+            found[ip] = city
+    return HttpResponse(json.dumps(found))
+
+
 def config_update(request):
     userid = request.session.get('userid', None)
     config_name = request.GET.get('name', '')

manage.py

@@ -1,15 +1,9 @@
 #!/usr/bin/env python
 import os
 import sys
-import multiprocessing
-import zoneresolver
 
 
 if __name__ == "__main__":
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnslog.settings")
     from django.core.management import execute_from_command_line
-    if len(sys.argv) >= 2 and sys.argv[1] == 'runserver':
-        p = multiprocessing.Process(target=zoneresolver.main)
-        p.daemon = True
-        p.start()
     execute_from_command_line(sys.argv)

templates/views.html

@@ -77,7 +77,7 @@
               <tbody>
                 {% for log in logs %}
                 <tr>
-                <th scope="row">{{log.id}}</th>
+                <td scope="row">{{log.id}}</td>
                 <td style="word-break:break-all">{{ log.host }}</td>
                 <td>{{ log.type }}</td>
                     <td><a href="?ip={{ log.ip }}">{% if log.ip %} {{ log.ip }}</a> {% else %} &nbsp; {% endif %} </td>
@@ -491,7 +491,27 @@
         $(function () {
             $('[data-toggle="popover"]').popover();
         })
-
+        $(document).ready(function() {
+            var allIPs = new Set([]);
+            $('.table tr').each(function(index) {
+                if (index !== 0){
+                    allIPs.add($(this).children('td')[3].innerText);
+                }
+            });
+            if (allIPs.size !== 0 ){
+                $.post("/ip2location", {"ips": Array.from(allIPs).join(',')}, function(result){
+                    var data = $.parseJSON(result);
+                    $('.table tr').each(function(index) {
+                        if (index !== 0){
+                            let ip = $(this).children('td')[3].innerText;
+                            if (ip in data){
+                                $(this).children('td')[4].innerText = data[ip];
+                            }
+                        }
+                    });
+                });
+            }
+        });
 </script>
 
 </body>

zoneresolver.py

@@ -1,13 +1,11 @@
 # -*- coding: utf-8 -*-
-
+import logging
 import os
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnslog.settings")
 import django
 django.setup()
 import copy
 import re
-import json
-import requests
 import struct
 import socket
 import random
@@ -15,6 +13,60 @@ from dnslib import RR, QTYPE, RCODE, TXT, A
 from dnslib.server import DNSServer, DNSHandler, BaseResolver, DNSLogger
 from logview.models import *
 from dnslog import settings
+import queue
+import threading
+from django.db import close_old_connections
+from django.utils import timezone
+
+
+q_query = queue.Queue()
+logger = logging.getLogger(__name__)
+
+logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S',
+                    filename='dns_server.log', filemode='a')
+
+
+def process_log():
+    global q_query
+    while True:
+        try:
+            user_domain, ip, domain, qtype, timestamp = q_query.get()
+            user = User.objects.filter(user_domain__exact=user_domain)
+            # 有需要也可以记录下不属于任何用戶的请求记录
+            # 你需要创建1个用户，然后把他的 user_domain 更新为 @
+            # if not user and domain.strip(".") != settings.ADMIN_DOMAIN:
+            #     user = User.objects.filter(user_domain__exact='@')
+            if not user:
+                logger.error('No such user: %s' % str(e), exc_info=True)
+
+            # 由于顺序获取客户端的IP地理位置过于耗时，大约300ms，不再顺序获取
+            # try:
+            #     doc = requests.get('https://whois.pconline.com.cn/ip.jsp?ip=%s' % ip, timeout=10.0).text.strip()
+            #     city = doc.split(' ')[0]
+            # except Exception as e:
+            #     city = ''
+
+            city = ''
+            try:
+                sub_name = domain.split('.')[-3-len(settings.DNS_DOMAIN.split('.'))]
+            except:
+                sub_name = ''
+            for _ in range(5):
+                try:
+                    log = DNSLog(user=user[0], host=domain.strip('.'), sub_name=sub_name, type=QTYPE[qtype], ip=ip,
+                                 city=city, created_time=timestamp)
+                    log.save()
+                    break
+                except django.db.utils.OperationalError as e:
+                    logger.error('process_log.exception.1: %s' % str(e), exc_info=True)
+                    close_old_connections()
+                except Exception as e:
+                    logger.error('process_log.exception.2: %s' % str(e), exc_info=True)
+        except django.db.utils.OperationalError as e:
+            close_old_connections()
+            logger.error('process_log.exception.3: %s' % str(e), exc_info=True)
+        except Exception as e:
+            logger.error('process_log.exception.4: %s' % str(e))
 
 
 class MySQLLogger:
@@ -40,6 +92,7 @@ class MySQLLogger:
         pass
 
     def log_request(self, handler, request):
+        global q_query
         if QTYPE[request.q.qtype] == 'AAAA':
             return
         domain = request.q.qname.__str__().lower()
@@ -48,20 +101,11 @@ class MySQLLogger:
         matches = re.search(r'\.?([^\.]+)\.%s\.' % settings.DNS_DOMAIN, domain)
         if not matches:
             return
-        user = User.objects.filter(user_domain__exact=matches.group(1))
+        user_domain = matches.group(1)
-        # 有需要可以记录
-        # if not user and domain.strip(".") != settings.ADMIN_DOMAIN:
-        #     user = User.objects.filter(user_domain__exact='@')
-        if not user:
-            return
         ip = handler.client_address[0]
-        try:
+        qtype = request.q.qtype
-            doc = requests.get('https://whois.pconline.com.cn/ip.jsp?ip=%s' % ip).text
+        item = (user_domain, ip, domain, qtype, timezone.now())
-            city = doc.split(' ')[0]
+        q_query.put(item)
-        except Exception as e:
-            city = ''
-        log = DNSLog(user=user[0], host=domain.strip('.'), type=QTYPE[request.q.qtype], ip=ip, city=city)
-        log.save()
 
     def log_send(self, handler, data):
         pass
@@ -71,31 +115,18 @@ class MySQLLogger:
 
 
 class ZoneResolver(BaseResolver):
-    """
-        Simple fixed zone file resolver.
-    """
-
     def __init__(self, zone, glob=False):
-        """
-            Initialise resolver from zone file.
-            Stores RRs as a list of (label,type,rr) tuples
-            If 'glob' is True use glob match against zone file
-        """
         self.zone = [(rr.rname, QTYPE[rr.rtype], rr) for rr in RR.fromZone(zone)]
         self.glob = glob
         self.eq = 'matchGlob' if glob else '__eq__'
 
     def resolve(self, request, handler):
-        """
-            Respond to DNS request - parameters are request packet & handler.
-            Method is expected to return DNS response
-        """
         reply = request.reply()
         qname = request.q.qname
         qtype = QTYPE[request.q.qtype]
         if qtype == 'TXT':
             reply.add_answer(RR(qname, QTYPE.TXT, rdata=TXT('Nothing to response')))
-        # rebind
+        # DNS rebind
         try:
             if qtype == 'A' and qname.__str__().endswith(settings.DNS_DOMAIN + '.'):
                 subs = qname.__str__().replace('.' + settings.DNS_DOMAIN + '.', '')
@@ -110,7 +141,7 @@ class ZoneResolver(BaseResolver):
                     reply.add_answer(rr)
                     return reply
         except Exception as e:
-            pass
+            logger.error('DNS rebind resolve.exception: %s' % str(e), exc_info=True)
         #
         for name, rtype, rr in self.zone:
             # Check if label & type match
@@ -143,6 +174,8 @@ def main():
 *.{dns_domain}.       IN      NS      {ns2_domain}.
 *.{dns_domain}.       IN      A       {server_ip}
 {dns_domain}.         IN      A       {server_ip}
+*.{dns_domain}.       IN      AAAA       2408:871a:2100:3:0:ff:b025:348d
+{dns_domain}.         IN      AAAA       2408:871a:2100:3:0:ff:b025:348d
 '''.format(
         dns_domain=settings.DNS_DOMAIN,
         ns1_domain=settings.NS1_DOMAIN,
@@ -150,10 +183,10 @@ def main():
         server_ip=settings.SERVER_IP
     )
     resolver = ZoneResolver(zone, True)
-    logger = MySQLLogger()
+    threading.Thread(target=process_log).start()
-    udp_server = DNSServer(resolver, port=53, address='', logger=logger)
+    udp_server = DNSServer(resolver, port=53, address='', logger=MySQLLogger())
     udp_server.start()
-    print("Zone Resolver started (%s:%d) [%s]" % ("*", 53, "UDP"))
+    logger.info("Zone Resolver started (%s:%d) [%s]" % ("*", 53, "UDP"))
 
 
 if __name__ == '__main__':