mirror of
https://github.com/shadow1ng/fscan.git
synced 2026-09-26 05:01:53 +08:00
feat: v2.1.0 核心重构与功能增强
## 架构重构
- 全局变量消除，迁移至 Config/State 对象
- SMB 插件融合（smb/smb2/smbghost/smbinfo）
- 服务探测重构，实现 Nmap 风格 fallback 机制
- 输出系统重构，TXT 实时刷盘 + 双写机制
- i18n 框架升级至 go-i18n
## 性能优化
- 正则表达式预编译
- 内存优化 map[string]struct{}
- 并发指纹匹配
- SOCKS5 连接复用
- 滑动窗口调度 + 自适应线程池
## 新功能
- Web 管理界面
- 多格式 POC 适配（xray/afrog）
- 增强指纹库（3139条）
- Favicon hash 指纹识别
- 插件选择性编译（Build Tags）
- fscan-lab 靶场环境
- 默认端口扩展（62→133）
## 构建系统
- 添加 no_local tag 支持排除本地插件
- 多版本构建：fscan/fscan-nolocal/fscan-web
- CI 添加 snapshot 模式支持仅测试构建
## Bug 修复
- 修复 120+ 个问题，包括 RDP panic、批量扫描漏报、
JSON 输出格式、Redis 检测、Context 超时等
## 测试增强
- 单元测试覆盖率 74-100%
- 并发安全测试
- 集成测试（Web/端口/服务/SSH/ICMP）
This commit is contained in:
ZacharyZcR
@@ -0,0 +1,513 @@
|
||||
package fingerprint
|
||||
|
||||
import (
|
||||
"crypto/md5" //nolint:gosec
|
||||
_ "embed"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
//go:embed web_fingerprint_v4.json
|
||||
var fingerprintHubData []byte
|
||||
|
||||
// EnhancedFingerprint FingerprintHub增强指纹结构
|
||||
type EnhancedFingerprint struct {
|
||||
ID string `json:"id"`
|
||||
Info struct {
|
||||
Name string `json:"name"`
|
||||
Author string `json:"author"`
|
||||
Tags string `json:"tags"`
|
||||
Severity string `json:"severity"`
|
||||
Metadata map[string]interface{} `json:"metadata"`
|
||||
} `json:"info"`
|
||||
HTTP []struct {
|
||||
Method string `json:"method"`
|
||||
Path []string `json:"path"`
|
||||
Matchers []struct {
|
||||
Type string `json:"type"`
|
||||
Words []string `json:"words"`
|
||||
Regex []string `json:"regex"`
|
||||
Hash []string `json:"hash"` // favicon hash
|
||||
Part string `json:"part"` // header, body
|
||||
CaseInsensitive bool `json:"case-insensitive"`
|
||||
Condition string `json:"condition"` // and, or
|
||||
} `json:"matchers"`
|
||||
} `json:"http"`
|
||||
}
|
||||
|
||||
// EnhancedFingerprintDB 增强指纹数据库
|
||||
type EnhancedFingerprintDB struct {
|
||||
Fingerprints []*EnhancedFingerprint
|
||||
// 预编译的正则表达式缓存
|
||||
regexCache map[string]*regexp.Regexp
|
||||
regexCacheMu sync.RWMutex // 保护regexCache的并发访问
|
||||
}
|
||||
|
||||
var (
|
||||
enhancedDB *EnhancedFingerprintDB
|
||||
enhancedDBOnce sync.Once // 保证只初始化一次
|
||||
)
|
||||
|
||||
// LoadEnhancedFingerprints 加载增强指纹库
|
||||
func LoadEnhancedFingerprints() error {
|
||||
var fps []*EnhancedFingerprint
|
||||
if err := json.Unmarshal(fingerprintHubData, &fps); err != nil {
|
||||
return fmt.Errorf("解析增强指纹库失败: %w", err)
|
||||
}
|
||||
|
||||
enhancedDB = &EnhancedFingerprintDB{
|
||||
Fingerprints: fps,
|
||||
regexCache: make(map[string]*regexp.Regexp),
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fingerprintMatch 指纹匹配结果（带优先级）
|
||||
type fingerprintMatch struct {
|
||||
Name string
|
||||
Priority int // 优先级分数，越高越优先
|
||||
}
|
||||
|
||||
// MatchEnhancedFingerprints 匹配增强指纹（并发版本，结果按优先级排序）
|
||||
func MatchEnhancedFingerprints(body []byte, headers string, favicon FaviconHashes) []string {
|
||||
// 使用 sync.Once 保证只初始化一次，线程安全
|
||||
enhancedDBOnce.Do(func() {
|
||||
_ = LoadEnhancedFingerprints() // 忽略错误，enhancedDB 为 nil 时下面会返回 nil
|
||||
})
|
||||
|
||||
if enhancedDB == nil || len(enhancedDB.Fingerprints) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
bodyStr := string(body)
|
||||
fingerprints := enhancedDB.Fingerprints
|
||||
total := len(fingerprints)
|
||||
|
||||
// 根据 CPU 核心数决定并发数，但不超过指纹数量
|
||||
workers := runtime.NumCPU()
|
||||
if workers > total {
|
||||
workers = total
|
||||
}
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
|
||||
// 每个 worker 处理的指纹数量
|
||||
chunkSize := (total + workers - 1) / workers
|
||||
|
||||
// 结果收集 channel（带优先级信息）
|
||||
resultCh := make(chan fingerprintMatch, total)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// 启动 worker
|
||||
for i := 0; i < workers; i++ {
|
||||
start := i * chunkSize
|
||||
end := start + chunkSize
|
||||
if end > total {
|
||||
end = total
|
||||
}
|
||||
if start >= total {
|
||||
break
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(fps []*EnhancedFingerprint) {
|
||||
defer wg.Done()
|
||||
for _, fp := range fps {
|
||||
if len(fp.HTTP) == 0 {
|
||||
continue
|
||||
}
|
||||
httpRule := fp.HTTP[0]
|
||||
for _, matcher := range httpRule.Matchers {
|
||||
if matchMatcher(matcher, bodyStr, headers, favicon, enhancedDB.regexCache) {
|
||||
resultCh <- fingerprintMatch{
|
||||
Name: fp.Info.Name,
|
||||
Priority: calcPriority(fp, matcher.Type),
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}(fingerprints[start:end])
|
||||
}
|
||||
|
||||
// 等待所有 worker 完成后关闭 channel
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(resultCh)
|
||||
}()
|
||||
|
||||
// 收集结果
|
||||
var matches []fingerprintMatch
|
||||
for m := range resultCh {
|
||||
matches = append(matches, m)
|
||||
}
|
||||
|
||||
// 按优先级排序（降序）
|
||||
sort.Slice(matches, func(i, j int) bool {
|
||||
if matches[i].Priority != matches[j].Priority {
|
||||
return matches[i].Priority > matches[j].Priority
|
||||
}
|
||||
return matches[i].Name < matches[j].Name // 同优先级按名称排序
|
||||
})
|
||||
|
||||
// 提取名称
|
||||
result := make([]string, len(matches))
|
||||
for i, m := range matches {
|
||||
result[i] = m.Name
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// calcPriority 计算指纹优先级分数
|
||||
func calcPriority(fp *EnhancedFingerprint, matcherType string) int {
|
||||
priority := 0
|
||||
|
||||
// 匹配类型权重（favicon 最精确）
|
||||
switch matcherType {
|
||||
case "favicon":
|
||||
priority += 100
|
||||
case "regex":
|
||||
priority += 50
|
||||
case "word":
|
||||
priority += 30
|
||||
}
|
||||
|
||||
// verified 状态加分
|
||||
if fp.Info.Metadata != nil {
|
||||
if verified, ok := fp.Info.Metadata["verified"].(bool); ok && verified {
|
||||
priority += 20
|
||||
}
|
||||
}
|
||||
|
||||
return priority
|
||||
}
|
||||
|
||||
// matchMatcher 匹配单个matcher
|
||||
func matchMatcher(matcher struct {
|
||||
Type string `json:"type"`
|
||||
Words []string `json:"words"`
|
||||
Regex []string `json:"regex"`
|
||||
Hash []string `json:"hash"`
|
||||
Part string `json:"part"`
|
||||
CaseInsensitive bool `json:"case-insensitive"`
|
||||
Condition string `json:"condition"`
|
||||
}, body, headers string, favicon FaviconHashes, regexCache map[string]*regexp.Regexp) bool {
|
||||
|
||||
switch matcher.Type {
|
||||
case "word":
|
||||
return matchWords(matcher, body, headers)
|
||||
case "regex":
|
||||
return matchRegex(matcher, body, headers, regexCache)
|
||||
case "favicon":
|
||||
return matchFavicon(matcher, favicon)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// matchWords 匹配关键词
|
||||
func matchWords(matcher struct {
|
||||
Type string `json:"type"`
|
||||
Words []string `json:"words"`
|
||||
Regex []string `json:"regex"`
|
||||
Hash []string `json:"hash"`
|
||||
Part string `json:"part"`
|
||||
CaseInsensitive bool `json:"case-insensitive"`
|
||||
Condition string `json:"condition"`
|
||||
}, body, headers string) bool {
|
||||
|
||||
// 确定匹配目标
|
||||
target := body
|
||||
if matcher.Part == "header" {
|
||||
target = headers
|
||||
}
|
||||
|
||||
// 预处理搜索词，避免循环内重复转换
|
||||
searchWords := matcher.Words
|
||||
if matcher.CaseInsensitive {
|
||||
target = strings.ToLower(target)
|
||||
searchWords = make([]string, len(matcher.Words))
|
||||
for i, w := range matcher.Words {
|
||||
searchWords[i] = strings.ToLower(w)
|
||||
}
|
||||
}
|
||||
|
||||
// 默认condition为or
|
||||
isAnd := matcher.Condition == "and"
|
||||
matchCount := 0
|
||||
|
||||
for _, searchWord := range searchWords {
|
||||
if strings.Contains(target, searchWord) {
|
||||
if !isAnd {
|
||||
// OR条件：匹配任一即可
|
||||
return true
|
||||
}
|
||||
matchCount++
|
||||
} else if isAnd {
|
||||
// AND条件：任一不匹配即失败
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// AND条件：全部匹配
|
||||
return isAnd && matchCount == len(matcher.Words)
|
||||
}
|
||||
|
||||
// matchRegex 匹配正则表达式
|
||||
func matchRegex(matcher struct {
|
||||
Type string `json:"type"`
|
||||
Words []string `json:"words"`
|
||||
Regex []string `json:"regex"`
|
||||
Hash []string `json:"hash"`
|
||||
Part string `json:"part"`
|
||||
CaseInsensitive bool `json:"case-insensitive"`
|
||||
Condition string `json:"condition"`
|
||||
}, body, headers string, regexCache map[string]*regexp.Regexp) bool {
|
||||
|
||||
// 确定匹配目标
|
||||
target := body
|
||||
if matcher.Part == "header" {
|
||||
target = headers
|
||||
}
|
||||
|
||||
// 默认condition为or
|
||||
isAnd := matcher.Condition == "and"
|
||||
|
||||
for _, pattern := range matcher.Regex {
|
||||
// 从缓存获取或编译正则（线程安全）
|
||||
var re *regexp.Regexp
|
||||
|
||||
// 先尝试读取缓存（读锁）
|
||||
enhancedDB.regexCacheMu.RLock()
|
||||
re, exists := regexCache[pattern]
|
||||
enhancedDB.regexCacheMu.RUnlock()
|
||||
|
||||
if !exists {
|
||||
// 不存在，需要编译并写入缓存（写锁）
|
||||
enhancedDB.regexCacheMu.Lock()
|
||||
// Double-check：可能其他goroutine已经编译了
|
||||
re, exists = regexCache[pattern]
|
||||
if !exists {
|
||||
var err error
|
||||
if matcher.CaseInsensitive {
|
||||
re, err = regexp.Compile("(?i)" + pattern)
|
||||
} else {
|
||||
re, err = regexp.Compile(pattern)
|
||||
}
|
||||
if err != nil {
|
||||
enhancedDB.regexCacheMu.Unlock()
|
||||
continue
|
||||
}
|
||||
regexCache[pattern] = re
|
||||
}
|
||||
enhancedDB.regexCacheMu.Unlock()
|
||||
}
|
||||
|
||||
// 确保 re 不为 nil（防止并发场景下的 nil panic）
|
||||
if re != nil && re.MatchString(target) {
|
||||
if !isAnd {
|
||||
return true
|
||||
}
|
||||
} else if isAnd {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// AND条件需要全部匹配
|
||||
return isAnd && len(matcher.Regex) > 0
|
||||
}
|
||||
|
||||
// matchFavicon 匹配favicon hash（同时支持 mmh3 和 MD5 格式）
|
||||
func matchFavicon(matcher struct {
|
||||
Type string `json:"type"`
|
||||
Words []string `json:"words"`
|
||||
Regex []string `json:"regex"`
|
||||
Hash []string `json:"hash"`
|
||||
Part string `json:"part"`
|
||||
CaseInsensitive bool `json:"case-insensitive"`
|
||||
Condition string `json:"condition"`
|
||||
}, favicon FaviconHashes) bool {
|
||||
|
||||
if favicon.MMH3 == "" && favicon.MD5 == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, hash := range matcher.Hash {
|
||||
// 同时匹配 mmh3（数字格式）和 MD5（十六进制格式）
|
||||
if hash == favicon.MMH3 || hash == favicon.MD5 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// FaviconHashes 包含 mmh3 和 MD5 两种格式的 hash
|
||||
type FaviconHashes struct {
|
||||
MMH3 string // Shodan/FOFA 风格: 有符号32位整数
|
||||
MD5 string // 传统 MD5 十六进制
|
||||
}
|
||||
|
||||
// VersionInfo 版本提取结果
|
||||
type VersionInfo struct {
|
||||
Name string // 产品名称
|
||||
Version string // 版本号
|
||||
}
|
||||
|
||||
// 通用版本提取正则（预编译）
|
||||
var versionExtractors = []struct {
|
||||
pattern *regexp.Regexp
|
||||
name string // 产品名称（空表示从匹配中提取）
|
||||
}{
|
||||
// Server header: nginx/1.18.0, Apache/2.4.41, IIS/10.0
|
||||
{regexp.MustCompile(`(?i)(?:^|[\s,])(?P<name>nginx|apache|iis|lighttpd|openresty|tengine)[/\s]?(?P<ver>[\d.]+)`), ""},
|
||||
|
||||
// X-Powered-By: PHP/7.4.3, ASP.NET/4.0
|
||||
{regexp.MustCompile(`(?i)(?:x-powered-by[:\s]*)?(?P<name>php|asp\.net|express|servlet)[/\s]?(?P<ver>[\d.]+)`), ""},
|
||||
|
||||
// Generator meta: WordPress 6.0, Drupal 9.0
|
||||
{regexp.MustCompile(`(?i)(?:generator|powered[\s-]*by)["\s:]*(?P<name>wordpress|drupal|joomla|typo3|hugo|jekyll|ghost)[\s/]*(?P<ver>[\d.]+)?`), ""},
|
||||
|
||||
// jQuery/Vue/React
|
||||
{regexp.MustCompile(`(?i)(?P<name>jquery|vue|react|angular)[\s./-]*(?:v|version)?[\s]*(?P<ver>\d+(?:\.\d+)+)`), ""},
|
||||
|
||||
// 通用 version= 或 ver= 模式
|
||||
{regexp.MustCompile(`(?i)(?P<name>[a-z][\w-]*?)[\s_-]*(?:version|ver)[=:\s"']*(?P<ver>[\d]+(?:\.[\d]+)+)`), ""},
|
||||
|
||||
// Tomcat, WebLogic, WebSphere
|
||||
{regexp.MustCompile(`(?i)(?P<name>tomcat|weblogic|websphere|jetty|jboss|wildfly)[/\s-]*(?P<ver>[\d.]+)`), ""},
|
||||
|
||||
// 数据库版本
|
||||
{regexp.MustCompile(`(?i)(?P<name>mysql|mariadb|postgresql|mongodb|redis|elasticsearch)[/\s-]*(?P<ver>[\d.]+)`), ""},
|
||||
|
||||
// OpenSSL, OpenSSH
|
||||
{regexp.MustCompile(`(?i)(?P<name>openssl|openssh)[/\s-]*(?P<ver>[\d.]+[a-z]?)`), ""},
|
||||
}
|
||||
|
||||
// ExtractVersions 从 HTTP 响应中提取软件版本信息
|
||||
func ExtractVersions(body string, headers string) []VersionInfo {
|
||||
content := headers + "\n" + body
|
||||
var results []VersionInfo
|
||||
seen := make(map[string]struct{})
|
||||
|
||||
for _, extractor := range versionExtractors {
|
||||
matches := extractor.pattern.FindAllStringSubmatch(content, -1)
|
||||
for _, match := range matches {
|
||||
var name, version string
|
||||
|
||||
// 提取命名捕获组
|
||||
for i, groupName := range extractor.pattern.SubexpNames() {
|
||||
if i > 0 && i < len(match) {
|
||||
switch groupName {
|
||||
case "name":
|
||||
name = strings.ToLower(match[i])
|
||||
case "ver":
|
||||
version = match[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 跳过空结果或已存在的
|
||||
if name == "" || version == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
key := name + ":" + version
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
|
||||
results = append(results, VersionInfo{
|
||||
Name: name,
|
||||
Version: version,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// CalculateFaviconHashes 计算 favicon 的 hash（同时返回 mmh3 和 MD5）
|
||||
func CalculateFaviconHashes(data []byte) FaviconHashes {
|
||||
if len(data) == 0 {
|
||||
return FaviconHashes{}
|
||||
}
|
||||
|
||||
// mmh3: base64编码后计算（Shodan/FOFA 标准）
|
||||
b64 := base64.StdEncoding.EncodeToString(data)
|
||||
mmh3Hash := mmh3Hash32([]byte(b64))
|
||||
|
||||
// MD5: 直接计算原始数据
|
||||
//nolint:gosec
|
||||
md5Hash := md5.Sum(data)
|
||||
|
||||
return FaviconHashes{
|
||||
MMH3: fmt.Sprintf("%d", mmh3Hash),
|
||||
MD5: fmt.Sprintf("%x", md5Hash),
|
||||
}
|
||||
}
|
||||
|
||||
// mmh3Hash32 计算 MurmurHash3 32位 hash（有符号整数）
|
||||
func mmh3Hash32(data []byte) int32 {
|
||||
const (
|
||||
c1 uint32 = 0xcc9e2d51
|
||||
c2 uint32 = 0x1b873593
|
||||
seed uint32 = 0
|
||||
)
|
||||
|
||||
length := len(data)
|
||||
nblocks := length / 4
|
||||
h1 := seed
|
||||
|
||||
// 处理4字节块
|
||||
for i := 0; i < nblocks; i++ {
|
||||
k1 := uint32(data[i*4]) | uint32(data[i*4+1])<<8 |
|
||||
uint32(data[i*4+2])<<16 | uint32(data[i*4+3])<<24
|
||||
|
||||
k1 *= c1
|
||||
k1 = (k1 << 15) | (k1 >> 17)
|
||||
k1 *= c2
|
||||
|
||||
h1 ^= k1
|
||||
h1 = (h1 << 13) | (h1 >> 19)
|
||||
h1 = h1*5 + 0xe6546b64
|
||||
}
|
||||
|
||||
// 处理剩余字节
|
||||
tail := data[nblocks*4:]
|
||||
var k1 uint32
|
||||
switch len(tail) {
|
||||
case 3:
|
||||
k1 ^= uint32(tail[2]) << 16
|
||||
fallthrough
|
||||
case 2:
|
||||
k1 ^= uint32(tail[1]) << 8
|
||||
fallthrough
|
||||
case 1:
|
||||
k1 ^= uint32(tail[0])
|
||||
k1 *= c1
|
||||
k1 = (k1 << 15) | (k1 >> 17)
|
||||
k1 *= c2
|
||||
h1 ^= k1
|
||||
}
|
||||
|
||||
// 最终混合
|
||||
h1 ^= uint32(length)
|
||||
h1 ^= h1 >> 16
|
||||
h1 *= 0x85ebca6b
|
||||
h1 ^= h1 >> 13
|
||||
h1 *= 0xc2b2ae35
|
||||
h1 ^= h1 >> 16
|
||||
|
||||
return int32(h1)
|
||||
}
|
||||
Reference in New Issue
Block a user