diff --git a/README.md b/README.md index ba7c5f6..4dd0b59 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# eyes.sh 1.0.2 +# eyes.sh 1.0.3 View this doc in [English](README_EN.md). @@ -11,6 +11,7 @@ View this doc in [English](README_EN.md). ## Chang Log +* 2024-06-02： 修复在DNS查询中串联IP查询功能，高并发请求时，响应慢的问题 * 2022-04-09： 增加多语言支持，添加英文翻译 * 2022-04-06： 注册用户名，限制只允许使用字母数字 diff --git a/README_EN.md b/README_EN.md index f5f5fbb..62961f7 100644 --- a/README_EN.md +++ b/README_EN.md @@ -1,4 +1,4 @@ -# eyes.sh 1.0.2 +# eyes.sh 1.0.3 View this doc in [Chinese](README.md). @@ -19,6 +19,7 @@ Demo site [http://eyes.sh/](http://eyes.sh/) ## Chang Log +* 2024-06-02： Bug fixed, a sync IP to geo location search (by HTTP) made DNS Server response too slow, especially under high concurrency queries * 2022-04-09： Add english translation Install diff --git a/dnslog/urls.py b/dnslog/urls.py index f51e9ed..726e403 100644 --- a/dnslog/urls.py +++ b/dnslog/urls.py @@ -22,6 +22,7 @@ urlpatterns = [ re_path(r'^rebind/$', views.rebind_view, name='rebind_view'), re_path(r'^rebind/gen$', views.rebind_gen, name='rebind_gen'), re_path(r'^as_admin/$', views.as_admin, name='as_admin'), + re_path(r'^ip2location$', views.ip_to_location, name='ip_to_location'), re_path(r'^config/update$', views.config_update, name='config_update'), path('i18n/', include('django.conf.urls.i18n')), re_path(r'^.*$', views.index, name='index'), diff --git a/logview/models.py b/logview/models.py index 94d9862..276599f 100644 --- a/logview/models.py +++ b/logview/models.py @@ -54,6 +54,7 @@ class DNSLog(models.Model): id = models.AutoField(primary_key=True) user = models.ForeignKey(User, on_delete=models.CASCADE) host = models.CharField(max_length=255, db_index=True, null=False) + sub_name = models.CharField(max_length=255, db_index=True, null=True) type = models.CharField(max_length=8, db_index=True, null=False) created_time = models.DateTimeField(auto_now_add=True) ip = models.GenericIPAddressField(null=False, db_index=True) @@ -64,7 +65,9 @@ class DNSLog(models.Model): class Meta: ordering = ['-id'] - + indexes = [ + models.Index(fields=['sub_name', 'user_id']), + ] class DNSLogAdmin(admin.ModelAdmin): list_display = ('user', 'host', 'type', 'created_time') diff --git a/logview/views.py b/logview/views.py index aefc2d0..2d4a279 100644 --- a/logview/views.py +++ b/logview/views.py @@ -20,11 +20,14 @@ from django.contrib.auth import authenticate, login from django.conf import settings from django.contrib.auth import logout from django.utils.translation import gettext as _ +from django.db.models import Q def get_city_by_ip(ip): try: - doc = requests.get('https://whois.pconline.com.cn/ip.jsp?ip=%s' % ip).text.strip() + doc = requests.get('https://whois.pconline.com.cn/ip.jsp?ip=%s' % ip, + headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'}).text.strip() city = doc.split(' ')[0] except Exception as e: city = '' @@ -402,12 +405,14 @@ def api(request, type, username, prefix): def group_api(request, username, prefix): token = request.GET.get('token', '') - if not User.objects.filter(username=username, token=token): + user = User.objects.filter(username=username, token=token) + if not user: return HttpResponse('Invalid token') postfix = ".%s.%s.%s" % (prefix, username, settings.DNS_DOMAIN) - res = DNSLog.objects.filter(host__endswith=postfix).order_by('host').values('host').distinct() + sub_name = prefix.split('.')[-1] + res = DNSLog.objects.filter(Q(sub_name=sub_name) & Q(user=user[0])).order_by('host').values('host').distinct()[:50] if res: - res = res[:50] + # res = res[:50] data = [item['host'].replace(postfix, '') for item in res] text = json.dumps({"success": "true", "data": data}) return HttpResponse(text, content_type="application/json") @@ -429,6 +434,21 @@ def as_admin(request): return redirect('/') + +@csrf_exempt +def ip_to_location(request): + ips = request.POST.get('ips', '') + if not ips: + return HttpResponse('') + matches = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ips) + found = {} + if matches: + for ip in matches[:10]: + city = get_city_by_ip(ip) + found[ip] = city + return HttpResponse(json.dumps(found)) + + def config_update(request): userid = request.session.get('userid', None) config_name = request.GET.get('name', '') diff --git a/manage.py b/manage.py index 55d9d53..5ffaa24 100644 --- a/manage.py +++ b/manage.py @@ -1,15 +1,9 @@ #!/usr/bin/env python import os import sys -import multiprocessing -import zoneresolver if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnslog.settings") from django.core.management import execute_from_command_line - if len(sys.argv) >= 2 and sys.argv[1] == 'runserver': - p = multiprocessing.Process(target=zoneresolver.main) - p.daemon = True - p.start() execute_from_command_line(sys.argv) diff --git a/templates/views.html b/templates/views.html index 61c2853..34af9bf 100644 --- a/templates/views.html +++ b/templates/views.html @@ -77,7 +77,7 @@ {% for log in logs %} - {{log.id}} + {{log.id}} {{ log.host }} {{ log.type }} {% if log.ip %} {{ log.ip }} {% else %}   {% endif %} @@ -491,7 +491,27 @@ $(function () { $('[data-toggle="popover"]').popover(); }) - + $(document).ready(function() { + var allIPs = new Set([]); + $('.table tr').each(function(index) { + if (index !== 0){ + allIPs.add($(this).children('td')[3].innerText); + } + }); + if (allIPs.size !== 0 ){ + $.post("/ip2location", {"ips": Array.from(allIPs).join(',')}, function(result){ + var data = $.parseJSON(result); + $('.table tr').each(function(index) { + if (index !== 0){ + let ip = $(this).children('td')[3].innerText; + if (ip in data){ + $(this).children('td')[4].innerText = data[ip]; + } + } + }); + }); + } + }); diff --git a/zoneresolver.py b/zoneresolver.py index d5d6769..507e1a9 100644 --- a/zoneresolver.py +++ b/zoneresolver.py @@ -1,13 +1,11 @@ # -*- coding: utf-8 -*- - +import logging import os os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dnslog.settings") import django django.setup() import copy import re -import json -import requests import struct import socket import random @@ -15,6 +13,60 @@ from dnslib import RR, QTYPE, RCODE, TXT, A from dnslib.server import DNSServer, DNSHandler, BaseResolver, DNSLogger from logview.models import * from dnslog import settings +import queue +import threading +from django.db import close_old_connections +from django.utils import timezone + + +q_query = queue.Queue() +logger = logging.getLogger(__name__) + +logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S', + filename='dns_server.log', filemode='a') + + +def process_log(): + global q_query + while True: + try: + user_domain, ip, domain, qtype, timestamp = q_query.get() + user = User.objects.filter(user_domain__exact=user_domain) + # 有需要也可以记录下不属于任何用戶的请求记录 + # 你需要创建1个用户，然后把他的 user_domain 更新为 @ + # if not user and domain.strip(".") != settings.ADMIN_DOMAIN: + # user = User.objects.filter(user_domain__exact='@') + if not user: + logger.error('No such user: %s' % str(e), exc_info=True) + + # 由于顺序获取客户端的IP地理位置过于耗时，大约300ms，不再顺序获取 + # try: + # doc = requests.get('https://whois.pconline.com.cn/ip.jsp?ip=%s' % ip, timeout=10.0).text.strip() + # city = doc.split(' ')[0] + # except Exception as e: + # city = '' + + city = '' + try: + sub_name = domain.split('.')[-3-len(settings.DNS_DOMAIN.split('.'))] + except: + sub_name = '' + for _ in range(5): + try: + log = DNSLog(user=user[0], host=domain.strip('.'), sub_name=sub_name, type=QTYPE[qtype], ip=ip, + city=city, created_time=timestamp) + log.save() + break + except django.db.utils.OperationalError as e: + logger.error('process_log.exception.1: %s' % str(e), exc_info=True) + close_old_connections() + except Exception as e: + logger.error('process_log.exception.2: %s' % str(e), exc_info=True) + except django.db.utils.OperationalError as e: + close_old_connections() + logger.error('process_log.exception.3: %s' % str(e), exc_info=True) + except Exception as e: + logger.error('process_log.exception.4: %s' % str(e)) class MySQLLogger: @@ -40,6 +92,7 @@ class MySQLLogger: pass def log_request(self, handler, request): + global q_query if QTYPE[request.q.qtype] == 'AAAA': return domain = request.q.qname.__str__().lower() @@ -48,20 +101,11 @@ class MySQLLogger: matches = re.search(r'\.?([^\.]+)\.%s\.' % settings.DNS_DOMAIN, domain) if not matches: return - user = User.objects.filter(user_domain__exact=matches.group(1)) - # 有需要可以记录 - # if not user and domain.strip(".") != settings.ADMIN_DOMAIN: - # user = User.objects.filter(user_domain__exact='@') - if not user: - return + user_domain = matches.group(1) ip = handler.client_address[0] - try: - doc = requests.get('https://whois.pconline.com.cn/ip.jsp?ip=%s' % ip).text - city = doc.split(' ')[0] - except Exception as e: - city = '' - log = DNSLog(user=user[0], host=domain.strip('.'), type=QTYPE[request.q.qtype], ip=ip, city=city) - log.save() + qtype = request.q.qtype + item = (user_domain, ip, domain, qtype, timezone.now()) + q_query.put(item) def log_send(self, handler, data): pass @@ -71,31 +115,18 @@ class MySQLLogger: class ZoneResolver(BaseResolver): - """ - Simple fixed zone file resolver. - """ - def __init__(self, zone, glob=False): - """ - Initialise resolver from zone file. - Stores RRs as a list of (label,type,rr) tuples - If 'glob' is True use glob match against zone file - """ self.zone = [(rr.rname, QTYPE[rr.rtype], rr) for rr in RR.fromZone(zone)] self.glob = glob self.eq = 'matchGlob' if glob else '__eq__' def resolve(self, request, handler): - """ - Respond to DNS request - parameters are request packet & handler. - Method is expected to return DNS response - """ reply = request.reply() qname = request.q.qname qtype = QTYPE[request.q.qtype] if qtype == 'TXT': reply.add_answer(RR(qname, QTYPE.TXT, rdata=TXT('Nothing to response'))) - # rebind + # DNS rebind try: if qtype == 'A' and qname.__str__().endswith(settings.DNS_DOMAIN + '.'): subs = qname.__str__().replace('.' + settings.DNS_DOMAIN + '.', '') @@ -110,7 +141,7 @@ class ZoneResolver(BaseResolver): reply.add_answer(rr) return reply except Exception as e: - pass + logger.error('DNS rebind resolve.exception: %s' % str(e), exc_info=True) # for name, rtype, rr in self.zone: # Check if label & type match @@ -143,6 +174,8 @@ def main(): *.{dns_domain}. IN NS {ns2_domain}. *.{dns_domain}. IN A {server_ip} {dns_domain}. IN A {server_ip} +*.{dns_domain}. IN AAAA 2408:871a:2100:3:0:ff:b025:348d +{dns_domain}. IN AAAA 2408:871a:2100:3:0:ff:b025:348d '''.format( dns_domain=settings.DNS_DOMAIN, ns1_domain=settings.NS1_DOMAIN, @@ -150,10 +183,10 @@ def main(): server_ip=settings.SERVER_IP ) resolver = ZoneResolver(zone, True) - logger = MySQLLogger() - udp_server = DNSServer(resolver, port=53, address='', logger=logger) + threading.Thread(target=process_log).start() + udp_server = DNSServer(resolver, port=53, address='', logger=MySQLLogger()) udp_server.start() - print("Zone Resolver started (%s:%d) [%s]" % ("*", 53, "UDP")) + logger.info("Zone Resolver started (%s:%d) [%s]" % ("*", 53, "UDP")) if __name__ == '__main__':