package core import ( "context" "crypto/tls" "fmt" "net" "net/http" "net/http/httptest" "strconv" "sync" "testing" "time" "github.com/shadow1ng/fscan/common" ) /* web_scanner_test.go - WebScanner核心逻辑测试 注意:web_scanner.go 包含网络IO和缓存管理。 本测试文件专注于可测试的纯逻辑和算法正确性: 1. IsWebServiceByFingerprint - Web服务识别逻辑 2. createTargetFromURL - URL解析和HostInfo构建 3. 缓存操作 - MarkAsWebService, GetWebServiceInfo, IsMarkedWebService 4. 指纹缓存 - SetFingerprints, GetFingerprints 不测试的部分(需要集成测试): - tryHTTP, DetectHTTPServiceOnly - 网络IO - Execute - 完整流程 "服务识别和URL解析是纯逻辑,应该测试。 缓存操作需要验证并发安全性。" */ func TestDetectHTTPServiceOnlyContextHonorsCancellation(t *testing.T) { cfg := common.GetGlobalConfig() oldTimeout := cfg.Network.WebTimeout cfg.Network.WebTimeout = 2 * time.Second defer func() { cfg.Network.WebTimeout = oldTimeout }() session := common.NewScanSession(cfg, common.NewState(), common.GetFlagVars()) ctx, cancel := context.WithCancel(context.Background()) cancel() start := time.Now() detected := GetWebPortDetector().DetectHTTPServiceOnlyContext(ctx, "203.0.113.1", 80, cfg, session) if detected { t.Fatal("canceled web detection should not report a service") } if elapsed := time.Since(start); elapsed > 200*time.Millisecond { t.Fatalf("canceled web detection took %s", elapsed) } } // ============================================================================= // 核心逻辑测试:Web服务识别 // ============================================================================= // TestIsWebServiceByFingerprint 测试Web服务识别逻辑 func TestIsWebServiceByFingerprint(t *testing.T) { tests := []struct { name string serviceInfo *ServiceInfo expected bool }{ { name: "nil服务信息", serviceInfo: nil, expected: false, }, { name: "空服务名", serviceInfo: &ServiceInfo{ Name: "", }, expected: false, }, { name: "HTTP服务", serviceInfo: &ServiceInfo{ Name: "http", }, expected: true, }, { name: "HTTPS服务", serviceInfo: &ServiceInfo{ Name: "https", }, expected: true, }, { name: "Nginx服务", serviceInfo: &ServiceInfo{ Name: "nginx", }, expected: true, }, { name: "Apache服务", serviceInfo: &ServiceInfo{ Name: "apache", }, expected: true, }, { name: "IIS服务", serviceInfo: &ServiceInfo{ Name: "iis", }, expected: true, }, { name: "Tomcat服务", serviceInfo: &ServiceInfo{ Name: "tomcat", }, expected: true, }, { name: "MySQL服务-非Web", serviceInfo: &ServiceInfo{ Name: "mysql", }, expected: false, }, { name: "Redis服务-非Web", serviceInfo: &ServiceInfo{ Name: "redis", }, expected: false, }, { name: "SSH服务-非Web", serviceInfo: &ServiceInfo{ Name: "ssh", }, expected: false, }, { name: "FTP服务-非Web", serviceInfo: &ServiceInfo{ Name: "ftp", }, expected: false, }, { name: "大小写混合-HTTP", serviceInfo: &ServiceInfo{ Name: "HTTP/1.1", }, expected: true, }, { name: "包含Web关键字-http-server", serviceInfo: &ServiceInfo{ Name: "custom-http-server", }, expected: true, }, { name: "Banner包含Server头", serviceInfo: &ServiceInfo{ Name: "unknown", Banner: "Server: Apache/2.4.41", }, expected: true, }, { name: "Banner包含HTTP协议", serviceInfo: &ServiceInfo{ Name: "unknown", Banner: "HTTP/1.1 200 OK", }, expected: true, }, { name: "Banner包含Content-Type", serviceInfo: &ServiceInfo{ Name: "unknown", Banner: "Content-Type: text/html", }, expected: true, }, { name: "Banner大写-SERVER", serviceInfo: &ServiceInfo{ Name: "unknown", Banner: "SERVER: NGINX/1.18.0", }, expected: true, }, { name: "非Web服务名+非Web Banner", serviceInfo: &ServiceInfo{ Name: "telnet", Banner: "Telnet Server Ready", }, expected: false, }, { name: "未知服务+无Banner", serviceInfo: &ServiceInfo{ Name: "unknown", Banner: "", }, expected: false, }, { name: "PHP服务", serviceInfo: &ServiceInfo{ Name: "php", }, expected: true, }, { name: "JSP服务", serviceInfo: &ServiceInfo{ Name: "jsp", }, expected: true, }, { name: "ASP服务", serviceInfo: &ServiceInfo{ Name: "asp", }, expected: true, }, { name: "SSL/TLS服务", serviceInfo: &ServiceInfo{ Name: "ssl", }, expected: true, }, { name: "包含非Web关键字-postgresql", serviceInfo: &ServiceInfo{ Name: "postgresql-server", }, expected: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := IsWebServiceByFingerprint(tt.serviceInfo) if result != tt.expected { t.Errorf("IsWebServiceByFingerprint() = %v, 期望 %v (Name=%q, Banner=%q)", result, tt.expected, tt.serviceInfo.Name, tt.serviceInfo.Banner) } }) } } // ============================================================================= // URL解析测试 // ============================================================================= // TestCreateTargetFromURL 测试URL解析和HostInfo构建 func TestCreateTargetFromURL(t *testing.T) { strategy := NewWebScanStrategy() tests := []struct { name string baseInfo common.HostInfo urlStr string expectNil bool expectedHost string expectedPort int expectedURL string }{ { name: "完整HTTP URL", baseInfo: common.HostInfo{}, urlStr: "http://example.com", expectNil: false, expectedHost: "example.com", expectedPort: 80, expectedURL: "http://example.com", }, { name: "完整HTTPS URL", baseInfo: common.HostInfo{}, urlStr: "https://example.com", expectNil: false, expectedHost: "example.com", expectedPort: 443, expectedURL: "https://example.com", }, { name: "HTTP+自定义端口", baseInfo: common.HostInfo{}, urlStr: "http://example.com:8080", expectNil: false, expectedHost: "example.com", expectedPort: 8080, expectedURL: "http://example.com:8080", }, { name: "HTTPS+自定义端口", baseInfo: common.HostInfo{}, urlStr: "https://example.com:8443", expectNil: false, expectedHost: "example.com", expectedPort: 8443, expectedURL: "https://example.com:8443", }, { name: "无协议头-自动添加http", baseInfo: common.HostInfo{}, urlStr: "example.com", expectNil: false, expectedHost: "example.com", expectedPort: 80, expectedURL: "http://example.com", }, { name: "无协议头+端口", baseInfo: common.HostInfo{}, urlStr: "example.com:8080", expectNil: false, expectedHost: "example.com", expectedPort: 8080, expectedURL: "http://example.com:8080", }, { name: "IP地址", baseInfo: common.HostInfo{}, urlStr: "http://192.168.1.1", expectNil: false, expectedHost: "192.168.1.1", expectedPort: 80, expectedURL: "http://192.168.1.1", }, { name: "IP地址+端口", baseInfo: common.HostInfo{}, urlStr: "http://192.168.1.1:8080", expectNil: false, expectedHost: "192.168.1.1", expectedPort: 8080, expectedURL: "http://192.168.1.1:8080", }, { name: "带路径的URL", baseInfo: common.HostInfo{}, urlStr: "http://example.com/path/to/page", expectNil: false, expectedHost: "example.com", expectedPort: 80, expectedURL: "http://example.com/path/to/page", }, { name: "带查询参数的URL", baseInfo: common.HostInfo{}, urlStr: "http://example.com/?key=value", expectNil: false, expectedHost: "example.com", expectedPort: 80, expectedURL: "http://example.com/?key=value", }, { name: "继承baseInfo属性", baseInfo: common.HostInfo{ Info: []string{"info1", "info2"}, }, urlStr: "http://example.com", expectNil: false, expectedHost: "example.com", expectedPort: 80, expectedURL: "http://example.com", }, { name: "非法URL-无效字符", baseInfo: common.HostInfo{}, urlStr: "http://example.com:abc", expectNil: true, // 端口非法,解析失败 }, { name: "localhost", baseInfo: common.HostInfo{}, urlStr: "http://localhost:8080", expectNil: false, expectedHost: "localhost", expectedPort: 8080, expectedURL: "http://localhost:8080", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := strategy.createTargetFromURL(tt.baseInfo, tt.urlStr) // 验证是否为nil if tt.expectNil { if result != nil { t.Errorf("期望返回nil, 实际返回 %+v", result) } return } if result == nil { t.Fatal("不应返回nil") } // 验证Host if result.Host != tt.expectedHost { t.Errorf("Host = %q, 期望 %q", result.Host, tt.expectedHost) } // 验证Ports if result.Port != tt.expectedPort { t.Errorf("Port = %d, 期望 %d", result.Port, tt.expectedPort) } // 验证Url if result.URL != tt.expectedURL { t.Errorf("URL = %q, 期望 %q", result.URL, tt.expectedURL) } // 验证baseInfo属性继承 if len(tt.baseInfo.Info) > 0 { if len(result.Info) != len(tt.baseInfo.Info) { t.Errorf("Infostr未继承, 长度 = %d, 期望 %d", len(result.Info), len(tt.baseInfo.Info)) } } }) } } // ============================================================================= // 缓存管理测试 // ============================================================================= // TestWebServiceCache 测试Web服务缓存操作 func TestWebServiceCache(t *testing.T) { // 清空缓存 SetGlobalState(common.NewState()) t.Run("存储和读取", func(t *testing.T) { serviceInfo := &ServiceInfo{ Name: "http", Banner: "Apache/2.4.41", } // 标记Web服务 MarkAsWebService("192.168.1.1", 80, serviceInfo) // 验证IsMarkedWebService if !IsMarkedWebService("192.168.1.1", 80) { t.Error("IsMarkedWebService应返回true") } // 验证GetWebServiceInfo info, exists := GetWebServiceInfo("192.168.1.1", 80) if !exists { t.Error("GetWebServiceInfo应返回exists=true") } if info.Name != "http" { t.Errorf("Name = %q, 期望 'http'", info.Name) } }) t.Run("不存在的服务", func(t *testing.T) { if IsMarkedWebService("192.168.1.2", 80) { t.Error("不存在的服务应返回false") } info, exists := GetWebServiceInfo("192.168.1.2", 80) if exists { t.Error("不存在的服务应返回exists=false") } if info != nil { t.Error("不存在的服务应返回nil info") } }) t.Run("覆盖写入", func(t *testing.T) { serviceInfo1 := &ServiceInfo{Name: "http"} serviceInfo2 := &ServiceInfo{Name: "https"} MarkAsWebService("192.168.1.3", 80, serviceInfo1) MarkAsWebService("192.168.1.3", 80, serviceInfo2) info, _ := GetWebServiceInfo("192.168.1.3", 80) if info.Name != "https" { t.Errorf("覆盖后Name = %q, 期望 'https'", info.Name) } }) t.Run("不同端口独立存储", func(t *testing.T) { serviceInfo80 := &ServiceInfo{Name: "http"} serviceInfo443 := &ServiceInfo{Name: "https"} MarkAsWebService("192.168.1.4", 80, serviceInfo80) MarkAsWebService("192.168.1.4", 443, serviceInfo443) info80, _ := GetWebServiceInfo("192.168.1.4", 80) info443, _ := GetWebServiceInfo("192.168.1.4", 443) if info80.Name != "http" { t.Errorf("端口80的Name = %q, 期望 'http'", info80.Name) } if info443.Name != "https" { t.Errorf("端口443的Name = %q, 期望 'https'", info443.Name) } }) } // TestWebServiceCache_Concurrent 测试并发安全性 func TestWebServiceCache_Concurrent(t *testing.T) { // 清空缓存 SetGlobalState(common.NewState()) t.Run("不同key并发写入", func(t *testing.T) { var wg sync.WaitGroup numGoroutines := 100 // 并发写入不同端口 for i := 0; i < numGoroutines; i++ { wg.Add(1) go func(id int) { defer wg.Done() serviceInfo := &ServiceInfo{ Name: "http", } MarkAsWebService("192.168.1.1", id, serviceInfo) }(i) } wg.Wait() // 验证数据完整性 for i := 0; i < numGoroutines; i++ { if !IsMarkedWebService("192.168.1.1", i) { t.Errorf("端口 %d 应被标记", i) } } }) t.Run("同一key并发读写", func(t *testing.T) { // 这才是真正的race condition测试 var wg sync.WaitGroup numGoroutines := 100 const testHost = "192.168.1.100" const testPort = 80 // 同时读写同一个key for i := 0; i < numGoroutines; i++ { wg.Add(2) // 写goroutine go func(id int) { defer wg.Done() serviceInfo := &ServiceInfo{ Name: "http", Banner: fmt.Sprintf("writer-%d", id), } MarkAsWebService(testHost, testPort, serviceInfo) }(i) // 读goroutine go func() { defer wg.Done() info, exists := GetWebServiceInfo(testHost, testPort) // 不验证具体内容(因为写入顺序不确定) // 只验证不会panic或返回不一致的exists/info if exists && info == nil { t.Error("exists=true但info=nil,数据不一致") } }() } wg.Wait() // 验证最终状态一致 info, exists := GetWebServiceInfo(testHost, testPort) if !exists { t.Error("应该至少有一次写入成功") } if info == nil { t.Error("exists=true但info=nil") } }) } // ============================================================================= // 指纹缓存测试 // ============================================================================= // ============================================================================= // 边界情况测试 // ============================================================================= // TestCreateTargetFromURL_EdgeCases 测试URL解析边界情况 func TestCreateTargetFromURL_EdgeCases(t *testing.T) { strategy := NewWebScanStrategy() t.Run("空URL", func(t *testing.T) { result := strategy.createTargetFromURL(common.HostInfo{}, "") if result != nil { t.Fatalf("空URL应被拒绝,实际 %#v", result) } }) t.Run("只有协议", func(t *testing.T) { result := strategy.createTargetFromURL(common.HostInfo{}, "http://") if result != nil { t.Fatalf("空Host URL应被拒绝,实际 %#v", result) } }) t.Run("非法URL不会因nil session panic", func(t *testing.T) { result := strategy.createTargetFromURL(common.HostInfo{}, "http://[::1") if result != nil { t.Fatalf("非法URL应被拒绝,实际 %#v", result) } }) t.Run("越界端口", func(t *testing.T) { result := strategy.createTargetFromURL(common.HostInfo{}, "http://example.com:70000") if result != nil { t.Fatalf("越界端口应被拒绝,实际 %#v", result) } }) t.Run("非数字端口", func(t *testing.T) { result := strategy.createTargetFromURL(common.HostInfo{}, "http://example.com:bad") if result != nil { t.Fatalf("非数字端口应被拒绝,实际 %#v", result) } }) t.Run("空端口", func(t *testing.T) { result := strategy.createTargetFromURL(common.HostInfo{}, "http://example.com:") if result != nil { t.Fatalf("空端口应被拒绝,实际 %#v", result) } }) t.Run("特殊字符URL", func(t *testing.T) { result := strategy.createTargetFromURL(common.HostInfo{}, "http://例子.com") // 中文域名可能成功解析(IDN) if result == nil { t.Log("中文域名解析失败(预期行为)") } }) t.Run("IPv6地址", func(t *testing.T) { result := strategy.createTargetFromURL(common.HostInfo{}, "http://[::1]:8080") if result == nil { t.Error("IPv6地址应能正确解析") } else { if result.Host != "::1" { t.Errorf("IPv6 Host = %q, 期望 '::1'", result.Host) } if result.Port != 8080 { t.Errorf("IPv6 Ports = %q, 期望 '8080'", result.Port) } } }) t.Run("IPv6无端口使用默认端口", func(t *testing.T) { result := strategy.createTargetFromURL(common.HostInfo{}, "https://[::1]/") if result == nil { t.Fatal("IPv6无端口URL应能正确解析") } if result.Host != "::1" || result.Port != 443 { t.Fatalf("IPv6默认端口解析错误: %#v", result) } }) } // TestIsWebServiceByFingerprint_Priority 测试识别优先级 func TestIsWebServiceByFingerprint_Priority(t *testing.T) { t.Run("非Web服务名优先级高于Web Banner", func(t *testing.T) { // 服务名是mysql,但Banner包含Web特征 serviceInfo := &ServiceInfo{ Name: "mysql", Banner: "Server: Apache", } result := IsWebServiceByFingerprint(serviceInfo) if result { t.Error("非Web服务名应优先,即使Banner包含Web特征") } }) t.Run("Web服务名优先级高于非Web Banner", func(t *testing.T) { serviceInfo := &ServiceInfo{ Name: "http", Banner: "MySQL Server Ready", } result := IsWebServiceByFingerprint(serviceInfo) if !result { t.Error("Web服务名应优先,即使Banner包含非Web特征") } }) } // ============================================================================= // 协议检测测试 // ============================================================================= // TestDetectHTTPScheme 测试HTTP/HTTPS协议智能检测 func TestDetectHTTPScheme(t *testing.T) { // 设置WebTimeout避免测试超时 cfg := common.GetGlobalConfig() oldTimeout := cfg.Network.WebTimeout cfg.Network.WebTimeout = 2 * time.Second defer func() { cfg.Network.WebTimeout = oldTimeout }() session := common.NewScanSession(cfg, common.NewState(), common.GetFlagVars()) t.Run("HTTPS服务器检测", func(t *testing.T) { // 创建HTTPS测试服务器 server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() // 解析服务器地址 host, portStr, err := net.SplitHostPort(server.Listener.Addr().String()) if err != nil { t.Fatalf("解析服务器地址失败: %v", err) } port, _ := strconv.Atoi(portStr) // 测试检测 result := DetectHTTPScheme(host, port, cfg, session) if result != "https" { t.Errorf("DetectHTTPScheme() = %q, 期望 'https'", result) } }) t.Run("HTTP服务器检测", func(t *testing.T) { // 创建HTTP测试服务器 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() // 解析服务器地址 host, portStr, err := net.SplitHostPort(server.Listener.Addr().String()) if err != nil { t.Fatalf("解析服务器地址失败: %v", err) } port, _ := strconv.Atoi(portStr) // 测试检测 result := DetectHTTPScheme(host, port, cfg, session) if result != "http" { t.Errorf("DetectHTTPScheme() = %q, 期望 'http'", result) } }) t.Run("不存在的服务", func(t *testing.T) { // 使用127.0.0.1的一个未使用端口 result := DetectHTTPScheme("127.0.0.1", 65534, cfg, session) if result != "" { t.Errorf("不存在的服务应返回空字符串, 实际 %q", result) } }) t.Run("非Web服务端口", func(t *testing.T) { // 创建一个TCP监听器但不响应HTTP listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Skipf("无法创建监听器: %v", err) } defer func() { _ = listener.Close() }() // 启动一个接受连接但立即关闭的goroutine go func() { for { conn, err := listener.Accept() if err != nil { return } conn.Close() } }() // 解析端口 _, portStr, _ := net.SplitHostPort(listener.Addr().String()) port, _ := strconv.Atoi(portStr) // 测试检测 result := DetectHTTPScheme("127.0.0.1", port, cfg, session) if result != "" { t.Logf("非Web服务检测返回: %q (预期空字符串,但立即关闭连接可能被误判)", result) } }) t.Run("TLS版本兼容性", func(t *testing.T) { // 测试TLS 1.0兼容性(DetectHTTPScheme设置MinVersion为TLS 1.0) server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) server.TLS = &tls.Config{ MinVersion: tls.VersionTLS10, MaxVersion: tls.VersionTLS10, } server.StartTLS() defer server.Close() host, portStr, _ := net.SplitHostPort(server.Listener.Addr().String()) port, _ := strconv.Atoi(portStr) result := DetectHTTPScheme(host, port, cfg, session) if result != "https" { t.Errorf("TLS 1.0服务器应被检测为https, 实际 %q", result) } }) } func TestCreateHTTPClientUsesPerSessionProxy(t *testing.T) { cfgA := common.NewConfig() cfgA.Network.WebTimeout = time.Second cfgA.Network.HTTPProxy = "http://127.0.0.1:18080" sessionA := common.NewScanSession(cfgA, common.NewState(), &common.FlagVars{}) cfgB := common.NewConfig() cfgB.Network.WebTimeout = time.Second cfgB.Network.HTTPProxy = "http://127.0.0.1:28080" sessionB := common.NewScanSession(cfgB, common.NewState(), &common.FlagVars{}) clientA := createHTTPClient(cfgA, sessionA) clientB := createHTTPClient(cfgB, sessionB) if clientA == clientB { t.Fatal("createHTTPClient reused a process-wide client") } proxyA := proxyForTest(t, clientA) proxyB := proxyForTest(t, clientB) if proxyA == proxyB { t.Fatalf("proxy URLs should be per config, both were %q", proxyA) } if proxyA != "http://127.0.0.1:18080" { t.Fatalf("proxyA = %q, want http://127.0.0.1:18080", proxyA) } if proxyB != "http://127.0.0.1:28080" { t.Fatalf("proxyB = %q, want http://127.0.0.1:28080", proxyB) } } func TestCreateHTTPClientNormalizesHTTPProxyWithoutScheme(t *testing.T) { cfg := common.NewConfig() cfg.Network.WebTimeout = time.Second cfg.Network.HTTPProxy = "127.0.0.1:18080" session := common.NewScanSession(cfg, common.NewState(), &common.FlagVars{}) client := createHTTPClient(cfg, session) proxy := proxyForTest(t, client) if proxy != "http://127.0.0.1:18080" { t.Fatalf("proxy = %q, want http://127.0.0.1:18080", proxy) } } func proxyForTest(t *testing.T, client *http.Client) string { t.Helper() transport, ok := client.Transport.(*http.Transport) if !ok { t.Fatal("client transport is not *http.Transport") } if transport.Proxy == nil { t.Fatal("client proxy is nil") } req, err := http.NewRequest(http.MethodGet, "http://example.com", nil) if err != nil { t.Fatal(err) } proxyURL, err := transport.Proxy(req) if err != nil { t.Fatal(err) } if proxyURL == nil { t.Fatal("proxy URL is nil") } return proxyURL.String() }