mirror of
https://github.com/shadow1ng/fscan.git
synced 2026-09-22 19:21:52 +08:00
refactor: replace bloom filter with map for deduplication
Bloom filter has false positive risk which can silently drop valid scan results. Map provides exact deduplication with negligible memory overhead at the scale of open ports (typically thousands, not millions).
This commit is contained in:
ZacharyZcR
@@ -1,66 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"hash/fnv"
|
||||
)
|
||||
|
||||
// BloomFilter 布隆过滤器，用于ICMP包去重
|
||||
type BloomFilter struct {
|
||||
bits []bool
|
||||
size uint32
|
||||
k uint32 // hash函数数量
|
||||
}
|
||||
|
||||
// NewBloomFilter 创建布隆过滤器
|
||||
// size: 预期元素数量
|
||||
// falsePositiveRate: 期望的误判率（通常0.01即1%）
|
||||
func NewBloomFilter(size int, falsePositiveRate float64) *BloomFilter {
|
||||
// 计算最优bit数组大小: m = -n*ln(p) / (ln(2)^2)
|
||||
// 简化计算：m ≈ n * 10 for p=0.01
|
||||
m := uint32(size * 10)
|
||||
if m < 1024 {
|
||||
m = 1024 // 最小1KB
|
||||
}
|
||||
|
||||
// 计算最优hash函数数量: k = (m/n) * ln(2)
|
||||
// 简化：k ≈ 7 for p=0.01
|
||||
k := uint32(7)
|
||||
|
||||
return &BloomFilter{
|
||||
bits: make([]bool, m),
|
||||
size: m,
|
||||
k: k,
|
||||
}
|
||||
}
|
||||
|
||||
// Add 添加元素到过滤器
|
||||
func (bf *BloomFilter) Add(data string) {
|
||||
for i := uint32(0); i < bf.k; i++ {
|
||||
pos := bf.hash(data, i)
|
||||
bf.bits[pos] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Contains 检查元素是否可能存在
|
||||
// 返回true：可能存在（有误判可能）
|
||||
// 返回false：一定不存在
|
||||
func (bf *BloomFilter) Contains(data string) bool {
|
||||
for i := uint32(0); i < bf.k; i++ {
|
||||
pos := bf.hash(data, i)
|
||||
if !bf.bits[pos] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// hash 计算hash值
|
||||
func (bf *BloomFilter) hash(data string, seed uint32) uint32 {
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(data))
|
||||
// 添加seed实现多个hash函数
|
||||
for i := uint32(0); i < seed; i++ {
|
||||
_, _ = h.Write([]byte{byte(i)})
|
||||
}
|
||||
return h.Sum32() % bf.size
|
||||
}
|
||||
@@ -1,168 +0,0 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
/*
|
||||
bloom_filter_test.go - BloomFilter 高价值测试
|
||||
|
||||
测试重点：
|
||||
1. 基本正确性 - Add后Contains返回true，未添加的返回false
|
||||
2. 误判率验证 - 实际误判率应接近理论值(1%)
|
||||
3. 大规模数据 - 模拟真实ICMP去重场景
|
||||
|
||||
不测试：
|
||||
- 内部哈希实现细节
|
||||
- 精确的数学公式验证
|
||||
*/
|
||||
|
||||
// TestBloomFilter_BasicCorrectness 基本正确性测试
|
||||
func TestBloomFilter_BasicCorrectness(t *testing.T) {
|
||||
bf := NewBloomFilter(1000, 0.01)
|
||||
|
||||
// 添加元素后应该能找到
|
||||
testData := []string{
|
||||
"192.168.1.1",
|
||||
"10.0.0.1",
|
||||
"172.16.0.1",
|
||||
}
|
||||
|
||||
for _, data := range testData {
|
||||
bf.Add(data)
|
||||
}
|
||||
|
||||
for _, data := range testData {
|
||||
if !bf.Contains(data) {
|
||||
t.Errorf("已添加的元素 %s 应该返回 true", data)
|
||||
}
|
||||
}
|
||||
|
||||
// 未添加的元素（大概率）返回false
|
||||
notAdded := []string{
|
||||
"8.8.8.8",
|
||||
"1.1.1.1",
|
||||
"255.255.255.255",
|
||||
}
|
||||
|
||||
falsePositives := 0
|
||||
for _, data := range notAdded {
|
||||
if bf.Contains(data) {
|
||||
falsePositives++
|
||||
}
|
||||
}
|
||||
|
||||
// 3个未添加元素全部误判的概率极低（<0.0001%）
|
||||
if falsePositives == len(notAdded) {
|
||||
t.Error("所有未添加元素都返回true，布隆过滤器可能有问题")
|
||||
}
|
||||
}
|
||||
|
||||
// TestBloomFilter_FalsePositiveRate 误判率验证
|
||||
//
|
||||
// 对于 n=10000, p=0.01 的布隆过滤器：
|
||||
// 实际误判率应该在 0.5% - 2% 之间（允许统计波动）
|
||||
func TestBloomFilter_FalsePositiveRate(t *testing.T) {
|
||||
n := 10000 // 添加的元素数
|
||||
bf := NewBloomFilter(n, 0.01)
|
||||
|
||||
// 添加n个元素
|
||||
for i := 0; i < n; i++ {
|
||||
bf.Add(fmt.Sprintf("added_%d", i))
|
||||
}
|
||||
|
||||
// 测试n个未添加的元素
|
||||
falsePositives := 0
|
||||
testCount := n
|
||||
for i := 0; i < testCount; i++ {
|
||||
if bf.Contains(fmt.Sprintf("not_added_%d", i)) {
|
||||
falsePositives++
|
||||
}
|
||||
}
|
||||
|
||||
actualRate := float64(falsePositives) / float64(testCount)
|
||||
|
||||
// 允许的误判率范围：0.1% - 3%（考虑统计波动）
|
||||
if actualRate > 0.03 {
|
||||
t.Errorf("误判率过高: %.2f%% (期望 < 3%%)", actualRate*100)
|
||||
}
|
||||
|
||||
t.Logf("实际误判率: %.2f%% (%d/%d)", actualRate*100, falsePositives, testCount)
|
||||
}
|
||||
|
||||
// TestBloomFilter_LargeScale 大规模数据测试
|
||||
//
|
||||
// 模拟真实的ICMP去重场景：100万个IP地址
|
||||
func TestBloomFilter_LargeScale(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("跳过大规模测试")
|
||||
}
|
||||
|
||||
n := 1000000 // 100万
|
||||
bf := NewBloomFilter(n, 0.01)
|
||||
|
||||
// 添加100万个元素
|
||||
for i := 0; i < n; i++ {
|
||||
bf.Add(fmt.Sprintf("192.168.%d.%d", i/256, i%256))
|
||||
}
|
||||
|
||||
// 验证已添加的元素
|
||||
sampleSize := 1000
|
||||
for i := 0; i < sampleSize; i++ {
|
||||
idx := i * (n / sampleSize)
|
||||
data := fmt.Sprintf("192.168.%d.%d", idx/256, idx%256)
|
||||
if !bf.Contains(data) {
|
||||
t.Errorf("已添加的元素 %s 返回 false", data)
|
||||
}
|
||||
}
|
||||
|
||||
// 测试未添加元素的误判率
|
||||
falsePositives := 0
|
||||
for i := 0; i < sampleSize; i++ {
|
||||
if bf.Contains(fmt.Sprintf("10.%d.%d.%d", i/65536, (i/256)%256, i%256)) {
|
||||
falsePositives++
|
||||
}
|
||||
}
|
||||
|
||||
actualRate := float64(falsePositives) / float64(sampleSize)
|
||||
if actualRate > 0.03 {
|
||||
t.Errorf("大规模场景误判率过高: %.2f%%", actualRate*100)
|
||||
}
|
||||
|
||||
t.Logf("100万元素场景误判率: %.2f%%", actualRate*100)
|
||||
}
|
||||
|
||||
// TestBloomFilter_NoFalseNegative 验证无假阴性
|
||||
//
|
||||
// 布隆过滤器的核心保证：已添加的元素必定返回true
|
||||
func TestBloomFilter_NoFalseNegative(t *testing.T) {
|
||||
bf := NewBloomFilter(10000, 0.01)
|
||||
|
||||
// 添加5000个元素
|
||||
added := make([]string, 5000)
|
||||
for i := range added {
|
||||
added[i] = fmt.Sprintf("element_%d", i)
|
||||
bf.Add(added[i])
|
||||
}
|
||||
|
||||
// 全部验证
|
||||
for _, data := range added {
|
||||
if !bf.Contains(data) {
|
||||
t.Fatalf("假阴性！已添加的元素 %s 返回 false", data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBloomFilter_EmptyFilter 空过滤器测试
|
||||
func TestBloomFilter_EmptyFilter(t *testing.T) {
|
||||
bf := NewBloomFilter(100, 0.01)
|
||||
|
||||
// 空过滤器应该对任何查询返回false
|
||||
testCases := []string{"anything", "192.168.1.1", ""}
|
||||
for _, tc := range testCases {
|
||||
if bf.Contains(tc) {
|
||||
t.Errorf("空过滤器对 %q 返回 true", tc)
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-5
@@ -328,8 +328,8 @@ func RunIcmp1(hostslist []string, conn *icmp.PacketConn, chanHosts chan string,
|
||||
var endflag atomic.Bool
|
||||
var listenerWg sync.WaitGroup
|
||||
|
||||
// 创建布隆过滤器用于去重（自动根据主机数量调整大小）
|
||||
bloomFilter := NewBloomFilter(len(hostslist), 0.01)
|
||||
// 去重集合：过滤重复的ICMP响应
|
||||
seen := make(map[string]struct{}, len(hostslist))
|
||||
|
||||
// 启动监听协程
|
||||
listenerWg.Add(1)
|
||||
@@ -365,11 +365,10 @@ func RunIcmp1(hostslist []string, conn *icmp.PacketConn, chanHosts chan string,
|
||||
if sourceIP != nil && !endflag.Load() {
|
||||
ipStr := sourceIP.String()
|
||||
|
||||
// 使用布隆过滤器去重，过滤重复的ICMP响应和杂包
|
||||
if bloomFilter.Contains(ipStr) {
|
||||
if _, dup := seen[ipStr]; dup {
|
||||
continue
|
||||
}
|
||||
bloomFilter.Add(ipStr)
|
||||
seen[ipStr] = struct{}{}
|
||||
|
||||
livewg.Add(1)
|
||||
select {
|
||||
|
||||
+11
-19
@@ -39,46 +39,38 @@ var resourceExhaustedPatterns = []string{
|
||||
}
|
||||
|
||||
// resultCollector 结果收集器，用于并发安全地收集扫描结果
|
||||
// 使用 Bloom Filter 去重 + slice 存储，大规模扫描时内存更优
|
||||
type resultCollector struct {
|
||||
mu sync.Mutex
|
||||
addrs []string
|
||||
bloom *BloomFilter
|
||||
stream chan<- string // 可选：流式通知 channel
|
||||
addrs map[string]struct{}
|
||||
stream chan<- string
|
||||
}
|
||||
|
||||
// newResultCollector 创建结果收集器
|
||||
func newResultCollector(stream chan<- string, expectedSize int) *resultCollector {
|
||||
if expectedSize < 1024 {
|
||||
expectedSize = 1024
|
||||
}
|
||||
func newResultCollector(stream chan<- string) *resultCollector {
|
||||
return &resultCollector{
|
||||
addrs: make([]string, 0, expectedSize/10),
|
||||
bloom: NewBloomFilter(expectedSize, 0.001),
|
||||
addrs: make(map[string]struct{}),
|
||||
stream: stream,
|
||||
}
|
||||
}
|
||||
|
||||
// Add 添加一个扫描结果
|
||||
func (c *resultCollector) Add(addr string) {
|
||||
c.mu.Lock()
|
||||
if c.bloom.Contains(addr) {
|
||||
if _, dup := c.addrs[addr]; dup {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
c.bloom.Add(addr)
|
||||
c.addrs = append(c.addrs, addr)
|
||||
c.addrs[addr] = struct{}{}
|
||||
c.mu.Unlock()
|
||||
if c.stream != nil {
|
||||
c.stream <- addr
|
||||
}
|
||||
}
|
||||
|
||||
// GetAll 获取所有结果
|
||||
func (c *resultCollector) GetAll() []string {
|
||||
c.mu.Lock()
|
||||
result := make([]string, len(c.addrs))
|
||||
copy(result, c.addrs)
|
||||
result := make([]string, 0, len(c.addrs))
|
||||
for addr := range c.addrs {
|
||||
result = append(result, addr)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
return result
|
||||
}
|
||||
@@ -189,7 +181,7 @@ func EnhancedPortScan(ctx context.Context, hosts []string, ports string, timeout
|
||||
to := time.Duration(timeout) * time.Second
|
||||
adaptiveTO := NewAdaptiveTimeout(to)
|
||||
var count int64
|
||||
collector := newResultCollector(stream, totalTasks)
|
||||
collector := newResultCollector(stream)
|
||||
failedCollector := &failedPortCollector{}
|
||||
var wg sync.WaitGroup
|
||||
|
||||
|
||||
Reference in New Issue
Block a user