archive/fscan
1
0
Fork 0
mirror of https://github.com/shadow1ng/fscan.git synced 2026-09-22 03:10:42 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Harden scan robustness and tests

Browse Source
This commit is contained in:
ZacharyZcR
2026-06-14 22:23:48 +08:00
parent 5ad914a1bb
commit c49c23c7f0
100 changed files with 4483 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webscan/lib/Eval.go
+41 -8
View File
@@ -33,6 +33,8 @@ var (
baseProgramOpt []cel.ProgramOption
)
const maxPOCResponseBodyBytes = 8 << 20
// 包级POC配置（atomic 保证并发安全）
var pocDNSLog atomic.Bool
@@ -386,6 +388,9 @@ func reverseCheck(r *Reverse, timeout int64) bool {
// RandomStr 生成指定长度的随机字符串
func RandomStr(randSource *rand.Rand, letterBytes string, n int) string {
if n <= 0 || letterBytes == "" {
return ""
}
const (
// 用 6 位比特表示一个字母索引
letterIdxBits = 6
@@ -471,9 +476,9 @@ func DoRequest(req *http.Request, redirect bool, session *common.ScanSession) (*
)
if redirect {
oResp, err = Client.Do(req)
oResp, err = requestClient(true).Do(req)
} else {
oResp, err = ClientNoRedirect.Do(req)
oResp, err = requestClient(false).Do(req)
}
// 标准TLS连接失败时，尝试国密TLS客户端
@@ -484,12 +489,16 @@ func DoRequest(req *http.Request, redirect bool, session *common.ScanSession) (*
}
}
if redirect {
if oResp2, err2 := ClientGM.Do(req); err2 == nil {
oResp, err = oResp2, nil
if clientGM := gmRequestClient(true); clientGM != nil {
if oResp2, err2 := clientGM.Do(req); err2 == nil {
oResp, err = oResp2, nil
}
}
} else {
if oResp2, err2 := ClientNoRedirectGM.Do(req); err2 == nil {
oResp, err = oResp2, nil
if clientGM := gmRequestClient(false); clientGM != nil {
if oResp2, err2 := clientGM.Do(req); err2 == nil {
oResp, err = oResp2, nil
}
}
}
}
@@ -513,6 +522,30 @@ func DoRequest(req *http.Request, redirect bool, session *common.ScanSession) (*
return resp, err
}
func requestClient(redirect bool) *http.Client {
if redirect {
if Client != nil {
return Client
}
return http.DefaultClient
}
if ClientNoRedirect != nil {
return ClientNoRedirect
}
return &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
}
}
func gmRequestClient(redirect bool) *http.Client {
if redirect {
return ClientGM
}
return ClientNoRedirectGM
}
// ParseURL 解析 TargetURL 并转换为自定义 TargetURL 类型
func ParseURL(u *url.URL) *UrlType {
return &UrlType{
@@ -597,7 +630,7 @@ func ParseResponse(oResp *http.Response) (*Response, error) {
// getRespBody 读取 HTTP 响应体并处理可能的 gzip 压缩
func getRespBody(oResp *http.Response) ([]byte, error) {
// 读取原始响应体
body, err := io.ReadAll(oResp.Body)
body, err := io.ReadAll(io.LimitReader(oResp.Body, maxPOCResponseBodyBytes))
if err != nil && !errors.Is(err, io.EOF) && len(body) == 0 {
return nil, err
}
@@ -610,7 +643,7 @@ func getRespBody(oResp *http.Response) ([]byte, error) {
}
defer func() { _ = reader.Close() }()
decompressed, err := io.ReadAll(reader)
decompressed, err := io.ReadAll(io.LimitReader(reader, maxPOCResponseBodyBytes))
if err != nil && !errors.Is(err, io.EOF) && len(decompressed) == 0 {
return nil, err
}
webscan/lib/eval_random.go
+43 -9
View File
@@ -1,6 +1,7 @@
package lib
import (
"fmt"
"math/rand" //nolint:gosec // G404: math/rand用于生成POC测试数据，非加密用途
"github.com/google/cel-go/checker/decls"
@@ -10,6 +11,8 @@ import (
exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
)
const maxRandomStringLength = 4096
// registerRandomDeclarations 注册随机函数的CEL声明
func registerRandomDeclarations() []*exprpb.Decl {
return []*exprpb.Decl{
@@ -46,12 +49,13 @@ func registerRandomImplementations() []*functions.Overload {
if !ok {
return types.ValOrErr(rhs, "unexpected type '%v' passed to randomInt", rhs.Type())
}
min, max := int(from), int(to)
if max <= min {
return types.NewErr("randomInt: max(%d) must be greater than min(%d)", max, min)
}
//nolint:gosec // G404: 用于生成POC测试随机数，非加密用途
return types.Int(rand.Intn(max-min) + min)
min, max := int64(from), int64(to)
span, err := randomIntSpan(min, max)
if err != nil {
return types.NewErr("%v", err)
}
//nolint:gosec // G404: 用于生成POC测试随机数，非加密用途
return types.Int(rand.Int63n(span) + min)
},
},
{
@@ -61,7 +65,11 @@ func registerRandomImplementations() []*functions.Overload {
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to randomLowercase", value.Type())
}
return types.String(randomLowercase(int(n)))
length, err := validateRandomStringLength(n)
if err != nil {
return types.NewErr("%v", err)
}
return types.String(randomLowercase(length))
},
},
{
@@ -71,7 +79,11 @@ func registerRandomImplementations() []*functions.Overload {
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to randomUppercase", value.Type())
}
return types.String(randomUppercase(int(n)))
length, err := validateRandomStringLength(n)
if err != nil {
return types.NewErr("%v", err)
}
return types.String(randomUppercase(length))
},
},
{
@@ -81,8 +93,30 @@ func registerRandomImplementations() []*functions.Overload {
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to randomString", value.Type())
}
return types.String(randomString(int(n)))
length, err := validateRandomStringLength(n)
if err != nil {
return types.NewErr("%v", err)
}
return types.String(randomString(length))
},
},
}
}
func randomIntSpan(min, max int64) (int64, error) {
if max <= min {
return 0, fmt.Errorf("randomInt: max(%d) must be greater than min(%d)", max, min)
}
const maxInt64 = int64(^uint64(0) >> 1)
if min < 0 && max > maxInt64+min {
return 0, fmt.Errorf("randomInt: range too large")
}
return max - min, nil
}
func validateRandomStringLength(n types.Int) (int, error) {
if n < 0 || n > maxRandomStringLength {
return 0, fmt.Errorf("random string length must be between 0 and %d", maxRandomStringLength)
}
return int(n), nil
}
webscan/lib/eval_string.go
+3 -3
View File
@@ -109,12 +109,12 @@ func registerStringImplementations() []*functions.Overload {
return types.NewErr("invalid length to 'substr'")
}
runes := []rune(str)
if start < 0 || length < 0 || int(start+length) > len(runes) {
if start < 0 || length < 0 || start > types.Int(len(runes)) || length > types.Int(len(runes))-start {
return types.NewErr("invalid start or length to 'substr'")
}
return types.String(runes[start : start+length])
return types.String(runes[int(start):int(start+length)])
}
return types.NewErr("too many arguments to 'substr'")
return types.NewErr("invalid argument count to 'substr'")
},
},
{
webscan/lib/eval_test.go
+134
View File
@@ -1,10 +1,12 @@
package lib
import (
"compress/gzip"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
@@ -258,6 +260,21 @@ func TestRandomInt(t *testing.T) {
}
}
func TestRandomIntRejectsOverflowingRange(t *testing.T) {
customLib := NewEnvOption()
env, err := NewEnv(&customLib)
if err != nil {
t.Fatalf("创建 CEL 环境失败: %v", err)
}
if _, err := Evaluate(env, "randomInt(-9223372036854775808, 9223372036854775807)", map[string]interface{}{}); err == nil {
t.Fatal("Evaluate() error = nil, want randomInt range error")
}
if _, err := randomIntSpan(-9223372036854775807-1, 9223372036854775807); err == nil {
t.Fatal("randomIntSpan() error = nil, want range too large")
}
}
func TestRandomLowercase(t *testing.T) {
customLib := NewEnvOption()
env, err := NewEnv(&customLib)
@@ -388,6 +405,26 @@ func TestRandomString(t *testing.T) {
}
}
func TestRandomStringLengthValidation(t *testing.T) {
customLib := NewEnvOption()
env, err := NewEnv(&customLib)
if err != nil {
t.Fatalf("创建 CEL 环境失败: %v", err)
}
for _, expr := range []string{
"randomLowercase(-1)",
fmt.Sprintf("randomUppercase(%d)", maxRandomStringLength+1),
fmt.Sprintf("randomString(%d)", maxRandomStringLength+1),
} {
t.Run(expr, func(t *testing.T) {
if _, err := Evaluate(env, expr, map[string]interface{}{}); err == nil {
t.Fatal("Evaluate() error = nil, want invalid random string length")
}
})
}
}
// =============================================================================
// eval_string.go 测试 - 字符串函数
// =============================================================================
@@ -469,6 +506,12 @@ func TestSubstr(t *testing.T) {
}
})
}
t.Run("长度溢出不panic", func(t *testing.T) {
if _, err := Evaluate(env, `substr("hello", 1, 9223372036854775807)`, map[string]interface{}{}); err == nil {
t.Fatal("Evaluate() error = nil, want invalid substr bounds")
}
})
}
func TestIStartsWith(t *testing.T) {
@@ -1086,6 +1129,45 @@ func TestGetRespBody(t *testing.T) {
}
}
func TestGetRespBodyLimitsPlainBody(t *testing.T) {
resp := &http.Response{
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(strings.Repeat("a", maxPOCResponseBodyBytes+1024))),
}
body, err := getRespBody(resp)
if err != nil {
t.Fatalf("getRespBody error = %v", err)
}
if len(body) != maxPOCResponseBodyBytes {
t.Fatalf("body len = %d, want %d", len(body), maxPOCResponseBodyBytes)
}
}
func TestGetRespBodyLimitsGzipBody(t *testing.T) {
var compressed strings.Builder
gzipWriter := gzip.NewWriter(&compressed)
if _, err := gzipWriter.Write([]byte(strings.Repeat("a", maxPOCResponseBodyBytes+1024))); err != nil {
t.Fatalf("gzip write error = %v", err)
}
if err := gzipWriter.Close(); err != nil {
t.Fatalf("gzip close error = %v", err)
}
resp := &http.Response{
Header: http.Header{"Content-Encoding": []string{"gzip"}},
Body: io.NopCloser(strings.NewReader(compressed.String())),
}
body, err := getRespBody(resp)
if err != nil {
t.Fatalf("getRespBody error = %v", err)
}
if len(body) != maxPOCResponseBodyBytes {
t.Fatalf("body len = %d, want %d", len(body), maxPOCResponseBodyBytes)
}
}
func TestDoRequestBuffersUnknownLengthBody(t *testing.T) {
previous := ClientNoRedirect
defer func() { ClientNoRedirect = previous }()
@@ -1124,6 +1206,52 @@ func TestDoRequestBuffersUnknownLengthBody(t *testing.T) {
}
}
func TestDoRequestUsesFallbackClientWhenGlobalClientNil(t *testing.T) {
previous := ClientNoRedirect
ClientNoRedirect = nil
defer func() { ClientNoRedirect = previous }()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("ok"))
}))
defer server.Close()
req, err := http.NewRequest(http.MethodGet, server.URL, nil)
if err != nil {
t.Fatalf("NewRequest error = %v", err)
}
resp, err := DoRequest(req, false, nil)
if err != nil {
t.Fatalf("DoRequest error = %v", err)
}
if string(resp.Body) != "ok" {
t.Fatalf("body = %q, want ok", resp.Body)
}
}
func TestDoRequestSkipsNilGMTLSFallback(t *testing.T) {
previousNR, previousGM := ClientNoRedirect, ClientNoRedirectGM
defer func() {
ClientNoRedirect = previousNR
ClientNoRedirectGM = previousGM
}()
ClientNoRedirect = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, errors.New("standard tls failed")
})}
ClientNoRedirectGM = nil
req, err := http.NewRequest(http.MethodGet, "https://example.com", nil)
if err != nil {
t.Fatalf("NewRequest error = %v", err)
}
if _, err := DoRequest(req, false, nil); err == nil {
t.Fatal("DoRequest expected standard TLS error")
}
}
func TestDoRequestReplaysBodyForGMTLSFallback(t *testing.T) {
previousNR, previousGM := ClientNoRedirect, ClientNoRedirectGM
defer func() {
@@ -1208,3 +1336,9 @@ func TestRandomStr(t *testing.T) {
})
}
}
func TestRandomStrRejectsNegativeLength(t *testing.T) {
if got := RandomStr(randSource, "abc", -1); got != "" {
t.Fatalf("RandomStr negative length = %q, want empty", got)
}
}
webscan/lib/poc_adapter.go
+130 -53
View File
@@ -24,6 +24,33 @@ const (
FormatUnknown PocFormat = "unknown"
)
type yamlStringList []string
func (l *yamlStringList) UnmarshalYAML(unmarshal func(interface{}) error) error {
var single string
if err := unmarshal(&single); err == nil {
if single != "" {
*l = []string{single}
}
return nil
}
var list []interface{}
if err := unmarshal(&list); err != nil {
return err
}
values := make([]string, 0, len(list))
for _, item := range list {
values = append(values, fmt.Sprintf("%v", item))
}
*l = values
return nil
}
func (l yamlStringList) String() string {
return strings.Join(l, ", ")
}
// UniversalPoc 通用POC接口 - 所有格式都要实现这个接口
type UniversalPoc interface {
GetName() string // 获取POC名称
@@ -144,29 +171,32 @@ func (f *FscanPocAdapter) ToFscanPoc() (*Poc, error) {
type NucleiPoc struct {
ID string `yaml:"id"`
Info struct {
Name string `yaml:"name"`
Author string `yaml:"author"`
Severity string `yaml:"severity"`
Description string `yaml:"description"`
Reference []string `yaml:"reference"`
Name string `yaml:"name"`
Author yamlStringList `yaml:"author"`
Severity string `yaml:"severity"`
Description string `yaml:"description"`
Reference yamlStringList `yaml:"reference"`
} `yaml:"info"`
HTTP []struct {
Method string `yaml:"method"`
Path []string `yaml:"path"`
Headers map[string]string `yaml:"headers"`
Body string `yaml:"body"`
Matchers []struct {
Type string `yaml:"type"`
Words []string `yaml:"words"`
Status []int `yaml:"status"`
Regex []string `yaml:"regex"`
Condition string `yaml:"condition"`
Part string `yaml:"part"`
} `yaml:"matchers"`
MatchersCondition string `yaml:"matchers-condition"`
Method string `yaml:"method"`
Path []string `yaml:"path"`
Headers map[string]string `yaml:"headers"`
Body string `yaml:"body"`
Matchers []NucleiMatcher `yaml:"matchers"`
MatchersCondition string `yaml:"matchers-condition"`
} `yaml:"http"`
}
type NucleiMatcher struct {
Type string `yaml:"type"`
Words []string `yaml:"words"`
Status []int `yaml:"status"`
Regex []string `yaml:"regex"`
Condition string `yaml:"condition"`
Part string `yaml:"part"`
Negative bool `yaml:"negative"`
}
// NucleiPocAdapter Nuclei格式适配器
type NucleiPocAdapter struct {
*NucleiPoc
@@ -198,9 +228,9 @@ func (n *NucleiPocAdapter) ToFscanPoc() (*Poc, error) {
poc := &Poc{
Name: n.GetName(),
Detail: Detail{
Author: n.Info.Author,
Author: n.Info.Author.String(),
Description: n.Info.Description,
Links: n.Info.Reference,
Links: []string(n.Info.Reference),
},
}
@@ -221,7 +251,7 @@ func (n *NucleiPocAdapter) ToFscanPoc() (*Poc, error) {
for _, path := range paths {
rule := Rules{
Method: method,
Path: path,
Path: normalizeNucleiPath(path),
Headers: httpReq.Headers,
Body: httpReq.Body,
}
@@ -246,26 +276,27 @@ func (n *NucleiPocAdapter) ToFscanPoc() (*Poc, error) {
return poc, nil
}
func normalizeNucleiPath(path string) string {
path = strings.TrimSpace(path)
path = strings.TrimPrefix(path, "{{BaseURL}}")
path = strings.TrimPrefix(path, "{{RootURL}}")
if path == "" {
return "/"
}
return path
}
// convertNucleiMatchers 转换Nuclei matchers为fscan expression
func convertNucleiMatchers(matchers []struct {
Type string `yaml:"type"`
Words []string `yaml:"words"`
Status []int `yaml:"status"`
Regex []string `yaml:"regex"`
Condition string `yaml:"condition"`
Part string `yaml:"part"`
}, matchersCondition string) string {
func convertNucleiMatchers(matchers []NucleiMatcher, matchersCondition string) string {
var conditions []string
for _, m := range matchers {
var matcherConds []string
switch m.Type {
switch strings.ToLower(m.Type) {
case "word":
for _, word := range m.Words {
// 转义双引号
escapedWord := strings.ReplaceAll(word, `"`, `\"`)
matcherConds = append(matcherConds, fmt.Sprintf(`response.body.bcontains(b"%s")`, escapedWord))
matcherConds = append(matcherConds, nucleiWordCondition(m.Part, word))
}
case "status":
for _, status := range m.Status {
@@ -273,9 +304,7 @@ func convertNucleiMatchers(matchers []struct {
}
case "regex":
for _, pattern := range m.Regex {
// 简化处理：直接用bmatches
escapedPattern := strings.ReplaceAll(pattern, `"`, `\"`)
matcherConds = append(matcherConds, fmt.Sprintf(`response.body.bmatches(b"%s")`, escapedPattern))
matcherConds = append(matcherConds, nucleiRegexCondition(m.Part, pattern))
}
case "dsl":
// DSL类型暂不支持，使用默认匹配
@@ -285,16 +314,20 @@ func convertNucleiMatchers(matchers []struct {
// 单个matcher内的条件组合
if len(matcherConds) > 0 {
connector := " && "
if m.Condition == "or" {
if strings.EqualFold(m.Condition, "or") {
connector = " || "
}
var combined string
if len(matcherConds) == 1 {
conditions = append(conditions, matcherConds[0])
combined = matcherConds[0]
} else {
combined := "(" + strings.Join(matcherConds, connector) + ")"
conditions = append(conditions, combined)
combined = "(" + strings.Join(matcherConds, connector) + ")"
}
if m.Negative {
combined = "!(" + combined + ")"
}
conditions = append(conditions, combined)
}
}
@@ -308,13 +341,57 @@ func convertNucleiMatchers(matchers []struct {
// 多个matcher之间的条件组合
connector := " && "
if matchersCondition == "or" {
if strings.EqualFold(matchersCondition, "or") {
connector = " || "
}
return strings.Join(conditions, connector)
}
func nucleiWordCondition(part, word string) string {
word = escapeCELBytesLiteral(word)
switch normalizeMatcherPart(part) {
case "header":
return fmt.Sprintf(`response.headers.exists(k, bytes(k + ": " + response.headers[k]).bcontains(b"%s"))`, word)
case "all":
return fmt.Sprintf(`(response.body.bcontains(b"%s") || response.headers.exists(k, bytes(k + ": " + response.headers[k]).bcontains(b"%s")))`, word, word)
default:
return fmt.Sprintf(`response.body.bcontains(b"%s")`, word)
}
}
func nucleiRegexCondition(part, pattern string) string {
pattern = escapeCELStringLiteral(pattern)
switch normalizeMatcherPart(part) {
case "header":
return fmt.Sprintf(`response.headers.exists(k, "%s".bmatches(bytes(k + ": " + response.headers[k])))`, pattern)
case "all":
return fmt.Sprintf(`("%s".bmatches(response.body) || response.headers.exists(k, "%s".bmatches(bytes(k + ": " + response.headers[k]))))`, pattern, pattern)
default:
return fmt.Sprintf(`"%s".bmatches(response.body)`, pattern)
}
}
func normalizeMatcherPart(part string) string {
switch strings.ToLower(strings.TrimSpace(part)) {
case "header", "headers", "all_headers":
return "header"
case "all":
return "all"
default:
return "body"
}
}
func escapeCELBytesLiteral(s string) string {
s = strings.ReplaceAll(s, `\`, `\\`)
return strings.ReplaceAll(s, `"`, `\"`)
}
func escapeCELStringLiteral(s string) string {
return escapeCELBytesLiteral(s)
}
// ============= xray格式适配器 =============
// XrayPoc xray POC结构
@@ -393,7 +470,7 @@ func (x *XrayPocAdapter) ToFscanPoc() (*Poc, error) {
// 展开 request 对象为 fscan Rule
fscanRule := Rules{
Method: rule.Request.Method,
Path: rule.Request.Path,
Path: normalizeNucleiPath(rule.Request.Path),
Headers: rule.Request.Headers,
Body: rule.Request.Body,
FollowRedirects: rule.Request.FollowRedirects,
@@ -426,14 +503,14 @@ func (x *XrayPocAdapter) ToFscanPoc() (*Poc, error) {
type AfrogPoc struct {
ID string `yaml:"id"`
Info struct {
Name string `yaml:"name"`
Author string `yaml:"author"`
Severity string `yaml:"severity"`
Verified bool `yaml:"verified"`
Description string `yaml:"description"`
Reference []string `yaml:"reference"`
Tags string `yaml:"tags"`
Created string `yaml:"created"`
Name string `yaml:"name"`
Author yamlStringList `yaml:"author"`
Severity string `yaml:"severity"`
Verified bool `yaml:"verified"`
Description string `yaml:"description"`
Reference yamlStringList `yaml:"reference"`
Tags string `yaml:"tags"`
Created string `yaml:"created"`
} `yaml:"info"`
Set map[string]interface{} `yaml:"set"`
Rules map[string]XrayRule `yaml:"rules"` // 复用 xray 的 rule 结构
@@ -472,9 +549,9 @@ func (a *AfrogPocAdapter) ToFscanPoc() (*Poc, error) {
poc := &Poc{
Name: a.GetName(),
Detail: Detail{
Author: a.Info.Author,
Author: a.Info.Author.String(),
Description: a.Info.Description,
Links: a.Info.Reference,
Links: []string(a.Info.Reference),
},
}
@@ -499,7 +576,7 @@ func (a *AfrogPocAdapter) ToFscanPoc() (*Poc, error) {
fscanRule := Rules{
Method: rule.Request.Method,
Path: rule.Request.Path,
Path: normalizeNucleiPath(rule.Request.Path),
Headers: rule.Request.Headers,
Body: rule.Request.Body,
FollowRedirects: rule.Request.FollowRedirects,
webscan/lib/poc_adapter_test.go
+225 -42
View File
@@ -3,6 +3,8 @@ package lib
import (
"strings"
"testing"
"github.com/google/cel-go/common/types"
)
// TestDetectPocFormat 测试POC格式检测
@@ -168,6 +170,9 @@ http:
if len(poc.Rules) != 2 {
t.Errorf("len(Poc.Rules) = %v, want %v", len(poc.Rules), 2)
}
if poc.Rules[0].Path != "/admin" || poc.Rules[1].Path != "/api" {
t.Fatalf("Nuclei paths = %q, %q; want /admin, /api", poc.Rules[0].Path, poc.Rules[1].Path)
}
if poc.Detail.Author != "pdteam" {
t.Errorf("Poc.Detail.Author = %v, want %v", poc.Detail.Author, "pdteam")
@@ -179,31 +184,101 @@ http:
}
}
func TestNormalizeNucleiPath(t *testing.T) {
tests := []struct {
in string
want string
}{
{"{{BaseURL}}", "/"},
{"{{BaseURL}}/admin", "/admin"},
{"{{RootURL}}/login", "/login"},
{" {{BaseURL}}/api?q=1 ", "/api?q=1"},
{"/plain", "/plain"},
}
for _, tt := range tests {
t.Run(tt.in, func(t *testing.T) {
if got := normalizeNucleiPath(tt.in); got != tt.want {
t.Fatalf("normalizeNucleiPath(%q) = %q, want %q", tt.in, got, tt.want)
}
})
}
}
func TestNucleiInfoAcceptsScalarAndListMetadata(t *testing.T) {
yaml := `
id: metadata-flex
info:
name: Metadata Flex
author:
- alice
- bob
reference: https://example.com/ref
http:
- path:
- "{{BaseURL}}"
`
adapter, err := loadNucleiPoc([]byte(yaml))
if err != nil {
t.Fatalf("loadNucleiPoc() error = %v", err)
}
poc, err := adapter.ToFscanPoc()
if err != nil {
t.Fatalf("ToFscanPoc() error = %v", err)
}
if poc.Detail.Author != "alice, bob" {
t.Fatalf("Author = %q, want alice, bob", poc.Detail.Author)
}
if len(poc.Detail.Links) != 1 || poc.Detail.Links[0] != "https://example.com/ref" {
t.Fatalf("Links = %#v, want scalar reference converted to slice", poc.Detail.Links)
}
}
func TestAfrogInfoAcceptsScalarAndListMetadata(t *testing.T) {
yaml := `
id: afrog-metadata-flex
info:
name: Afrog Metadata Flex
author: carol
reference:
- https://example.com/a
- https://example.com/b
rules:
r0:
request:
method: GET
path: "{{BaseURL}}/panel"
expression: response.status == 200
`
adapter, err := loadAfrogPoc([]byte(yaml))
if err != nil {
t.Fatalf("loadAfrogPoc() error = %v", err)
}
poc, err := adapter.ToFscanPoc()
if err != nil {
t.Fatalf("ToFscanPoc() error = %v", err)
}
if poc.Detail.Author != "carol" {
t.Fatalf("Author = %q, want carol", poc.Detail.Author)
}
if len(poc.Detail.Links) != 2 {
t.Fatalf("Links = %#v, want two references", poc.Detail.Links)
}
}
// TestConvertNucleiMatchers 测试Nuclei matcher转换
func TestConvertNucleiMatchers(t *testing.T) {
tests := []struct {
name string
matchers []struct {
Type string `yaml:"type"`
Words []string `yaml:"words"`
Status []int `yaml:"status"`
Regex []string `yaml:"regex"`
Condition string `yaml:"condition"`
Part string `yaml:"part"`
}
matchers []NucleiMatcher
matchersCondition string
wantContains string
}{
{
name: "单个word matcher",
matchers: []struct {
Type string `yaml:"type"`
Words []string `yaml:"words"`
Status []int `yaml:"status"`
Regex []string `yaml:"regex"`
Condition string `yaml:"condition"`
Part string `yaml:"part"`
}{
matchers: []NucleiMatcher{
{
Type: "word",
Words: []string{"admin"},
@@ -214,14 +289,7 @@ func TestConvertNucleiMatchers(t *testing.T) {
},
{
name: "单个status matcher",
matchers: []struct {
Type string `yaml:"type"`
Words []string `yaml:"words"`
Status []int `yaml:"status"`
Regex []string `yaml:"regex"`
Condition string `yaml:"condition"`
Part string `yaml:"part"`
}{
matchers: []NucleiMatcher{
{
Type: "status",
Status: []int{200},
@@ -232,14 +300,7 @@ func TestConvertNucleiMatchers(t *testing.T) {
},
{
name: "多个matcher - AND条件",
matchers: []struct {
Type string `yaml:"type"`
Words []string `yaml:"words"`
Status []int `yaml:"status"`
Regex []string `yaml:"regex"`
Condition string `yaml:"condition"`
Part string `yaml:"part"`
}{
matchers: []NucleiMatcher{
{
Type: "word",
Words: []string{"admin"},
@@ -254,14 +315,7 @@ func TestConvertNucleiMatchers(t *testing.T) {
},
{
name: "多个matcher - OR条件",
matchers: []struct {
Type string `yaml:"type"`
Words []string `yaml:"words"`
Status []int `yaml:"status"`
Regex []string `yaml:"regex"`
Condition string `yaml:"condition"`
Part string `yaml:"part"`
}{
matchers: []NucleiMatcher{
{
Type: "word",
Words: []string{"admin"},
@@ -299,6 +353,129 @@ func TestConvertNucleiMatchers(t *testing.T) {
}
}
func TestConvertNucleiMatchersEscapesCELByteLiterals(t *testing.T) {
matchers := []NucleiMatcher{
{
Type: "word",
Words: []string{`C:\Windows "System32"`},
},
{
Type: "regex",
Regex: []string{`admin\\d+"`},
},
}
expr := convertNucleiMatchers(matchers, "and")
if !strings.Contains(expr, `C:\\Windows \"System32\"`) {
t.Fatalf("word matcher was not escaped correctly: %s", expr)
}
if !strings.Contains(expr, `admin\\\\d+\"`) {
t.Fatalf("regex matcher was not escaped correctly: %s", expr)
}
if !strings.Contains(expr, `.bmatches(response.body)`) {
t.Fatalf("regex matcher should use pattern receiver and response body argument: %s", expr)
}
}
func TestConvertNucleiMatchersRespectsHeaderPart(t *testing.T) {
expr := convertNucleiMatchers([]NucleiMatcher{
{
Type: "word",
Words: []string{"nginx"},
Part: "header",
},
}, "")
result, err := Evaluate(GetBaseEnv(), expr, map[string]interface{}{
"response": &Response{
Headers: map[string]string{"Server": "nginx"},
Body: []byte("no match in body"),
},
})
if err != nil {
t.Fatalf("Evaluate(%q) error = %v", expr, err)
}
if result != types.True {
t.Fatalf("header matcher result = %v, want true; expr = %s", result, expr)
}
}
func TestConvertNucleiMatchersRespectsAllPart(t *testing.T) {
expr := convertNucleiMatchers([]NucleiMatcher{
{
Type: "regex",
Regex: []string{`JSESSIONID=\w+`},
Part: "all",
},
}, "")
result, err := Evaluate(GetBaseEnv(), expr, map[string]interface{}{
"response": &Response{
Headers: map[string]string{"Set-Cookie": "JSESSIONID=abc123"},
Body: []byte("no match in body"),
},
})
if err != nil {
t.Fatalf("Evaluate(%q) error = %v", expr, err)
}
if result != types.True {
t.Fatalf("all matcher result = %v, want true; expr = %s", result, expr)
}
}
func TestConvertNucleiMatchersRespectsNegative(t *testing.T) {
expr := convertNucleiMatchers([]NucleiMatcher{
{
Type: "word",
Words: []string{"error"},
Negative: true,
},
}, "")
result, err := Evaluate(GetBaseEnv(), expr, map[string]interface{}{
"response": &Response{Body: []byte("fatal error")},
})
if err != nil {
t.Fatalf("Evaluate(%q) error = %v", expr, err)
}
if result != types.False {
t.Fatalf("negative matcher result = %v, want false; expr = %s", result, expr)
}
}
func TestConvertNucleiMatchersConditionIsCaseInsensitive(t *testing.T) {
expr := convertNucleiMatchers([]NucleiMatcher{
{
Type: "word",
Words: []string{"alpha", "beta"},
Condition: "OR",
},
}, "AND")
if !strings.Contains(expr, " || ") {
t.Fatalf("matcher condition should be case-insensitive OR: %s", expr)
}
expr = convertNucleiMatchers([]NucleiMatcher{
{Type: "word", Words: []string{"alpha"}},
{Type: "word", Words: []string{"beta"}},
}, "OR")
if !strings.Contains(expr, " || ") {
t.Fatalf("matchers-condition should be case-insensitive OR: %s", expr)
}
}
func TestConvertNucleiMatchersTypeIsCaseInsensitive(t *testing.T) {
expr := convertNucleiMatchers([]NucleiMatcher{
{
Type: "WORD",
Words: []string{"admin"},
},
}, "")
if !strings.Contains(expr, `response.body.bcontains(b"admin")`) {
t.Fatalf("matcher type should be case-insensitive: %s", expr)
}
}
// contains 检查字符串是否包含子串
func contains(s, substr string) bool {
return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && hasSubstring(s, substr))
@@ -406,7 +583,7 @@ rules:
r1:
request:
method: GET
path: /admin/dashboard
path: "{{BaseURL}}/admin/dashboard"
headers:
Cookie: "{{cookie}}"
expression: response.status == 200
@@ -445,6 +622,9 @@ detail:
if poc.Rules[1].Headers["Cookie"] != `{{cookie}}` {
t.Errorf("Rules[1].Headers[Cookie] = %q, want %q", poc.Rules[1].Headers["Cookie"], `{{cookie}}`)
}
if poc.Rules[1].Path != "/admin/dashboard" {
t.Errorf("Rules[1].Path = %q, want /admin/dashboard", poc.Rules[1].Path)
}
}
// TestXrayNoOutput 测试 xray 没有 output 字段时 Search 为空（回归）
@@ -503,7 +683,7 @@ rules:
r1:
request:
method: GET
path: /panel
path: "{{BaseURL}}/panel"
headers:
Cookie: "{{sessid}}"
expression: response.status == 200 && response.body.bcontains(b"admin")
@@ -531,4 +711,7 @@ rules:
if poc.Rules[1].Headers["Cookie"] != `{{sessid}}` {
t.Errorf("Rules[1].Headers[Cookie] = %q, want %q", poc.Rules[1].Headers["Cookie"], `{{sessid}}`)
}
if poc.Rules[1].Path != "/panel" {
t.Errorf("Rules[1].Path = %q, want /panel", poc.Rules[1].Path)
}
}
webscan/lib/poc_executor.go
+9 -2
View File
@@ -359,7 +359,7 @@ func doSearch(re string, body string) map[string]string {
if len(result) > 1 && len(names) > 1 {
paramsMap := make(map[string]string)
for i, name := range names {
if i > 0 && i <= len(result) {
if i > 0 && i < len(result) && name != "" {
// 特殊处理Set-Cookie头：剥离Path/Expires等属性，仅保留key=value
if strings.HasPrefix(re, "Set-Cookie:") {
paramsMap[name] = optimizeCookies(result[i])
@@ -470,7 +470,7 @@ func clusterpoc(oReq *http.Request, p *Poc, variableMap map[string]interface{},
for comboIndex, paramCombo := range setsMap {
// Shiro Key测试特殊处理:默认只测试10个key
if p.Name == "poc-yaml-shiro-key" && !pocCtx.POCFull && comboIndex >= 10 {
if paramCombo[1] == "cbc" {
if shiroKeyMode(paramCombo) == "cbc" {
continue
}
if shiroKeyCount == 0 {
@@ -554,6 +554,13 @@ func clusterpoc(oReq *http.Request, p *Poc, variableMap map[string]interface{},
return success, nil
}
func shiroKeyMode(paramCombo []string) string {
if len(paramCombo) < 2 {
return ""
}
return paramCombo[1]
}
// applyParametersToRule 将参数应用到规则中，返回是否有替换发生和替换的参数列表
// 这是一个纯函数，不修改原始规则，而是修改传入的currentRule指针
func applyParametersToRule(
webscan/lib/poc_executor_test.go
+108
View File
@@ -425,3 +425,111 @@ func TestApplyParametersToRule(t *testing.T) {
})
}
}
func TestPocExecutorPureHelpers(t *testing.T) {
t.Run("isFuzz detects placeholders", func(t *testing.T) {
sets := ListMap{{Key: "token", Value: []string{"a", "b"}}}
if !isFuzz(Rules{Headers: map[string]string{"X-Token": "{{token}}"}}, sets) {
t.Fatal("header placeholder should require fuzzing")
}
if !isFuzz(Rules{Path: "/api/{{token}}"}, sets) {
t.Fatal("path placeholder should require fuzzing")
}
if !isFuzz(Rules{Body: "token={{token}}"}, sets) {
t.Fatal("body placeholder should require fuzzing")
}
if isFuzz(Rules{Path: "/api/static"}, sets) {
t.Fatal("static rule should not require fuzzing")
}
})
t.Run("Combo and MakeData", func(t *testing.T) {
if got := Combo(nil); got != nil {
t.Fatalf("Combo(nil) = %#v, want nil", got)
}
one := Combo(ListMap{{Key: "user", Value: []string{"admin", "root"}}})
if len(one) != 2 || one[0][0] != "admin" || one[1][0] != "root" {
t.Fatalf("single Combo = %#v", one)
}
combos := Combo(ListMap{
{Key: "user", Value: []string{"admin", "root"}},
{Key: "pass", Value: []string{"123", "456"}},
})
want := [][]string{{"admin", "123"}, {"root", "123"}, {"admin", "456"}, {"root", "456"}}
if !stringMatrixEqual(combos, want) {
t.Fatalf("Combo = %#v, want %#v", combos, want)
}
made := MakeData([][]string{{"b"}, {"c"}}, []string{"a"})
if !stringMatrixEqual(made, [][]string{{"a", "b"}, {"a", "c"}}) {
t.Fatalf("MakeData = %#v", made)
}
if got := shiroKeyMode([]string{"only-key"}); got != "" {
t.Fatalf("shiroKeyMode(short combo) = %q, want empty", got)
}
if got := shiroKeyMode([]string{"key", "cbc"}); got != "cbc" {
t.Fatalf("shiroKeyMode() = %q, want cbc", got)
}
})
t.Run("cloneRules deep-copies headers", func(t *testing.T) {
original := Rules{
Method: "POST",
Path: "/login",
Body: "a=b",
Search: "token",
FollowRedirects: true,
Expression: "true",
Headers: map[string]string{"X-Test": "one"},
Continue: true,
}
cloned := cloneRules(original)
cloned.Headers["X-Test"] = "two"
if original.Headers["X-Test"] != "one" {
t.Fatalf("cloneRules should deep copy headers, original = %#v", original.Headers)
}
if cloned.Method != original.Method || cloned.Path != original.Path || !cloned.FollowRedirects || !cloned.Continue {
t.Fatalf("cloneRules lost fields: %#v", cloned)
}
if cloneMap(nil) != nil {
t.Fatal("cloneMap(nil) should return nil")
}
})
t.Run("doSearch and GetHeader", func(t *testing.T) {
header := GetHeader(map[string]string{"Set-Cookie": "sid=abc; Path=/; HttpOnly", "Server": "nginx"})
if !strings.Contains(header, "Set-Cookie: sid=abc; Path=/; HttpOnly") || !strings.HasSuffix(header, "\r\n") {
t.Fatalf("GetHeader output = %q", header)
}
result := doSearch(`Set-Cookie:\s*(?P<cookie>[^\n]+)`, header)
if result["cookie"] != "sid=abc" {
t.Fatalf("cookie search = %#v", result)
}
result = doSearch(`token=(\w+)&id=(?P<id>\d+)`, "token=abc&id=42")
if result[""] != "" || result["id"] != "42" || len(result) != 1 {
t.Fatalf("unnamed groups should be skipped, got %#v", result)
}
if got := doSearch(`(?P<bad>`, "body"); got != nil {
t.Fatalf("invalid regex result = %#v, want nil", got)
}
if got := doSearch(`nomatch(?P<value>\d+)`, "body"); got != nil {
t.Fatalf("no match result = %#v, want nil", got)
}
})
}
func stringMatrixEqual(a, b [][]string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if len(a[i]) != len(b[i]) {
return false
}
for j := range a[i] {
if a[i][j] != b[i][j] {
return false
}
}
}
return true
}
webscan/web_scan.go
+23
View File
@@ -10,6 +10,7 @@ import (
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
@@ -115,6 +116,20 @@ func buildTargetURL(info *common.HostInfo) (string, error) {
if err != nil {
return "", fmt.Errorf("%w: %w", ErrInvalidURL, err)
}
if parsedURL.Hostname() == "" {
return "", fmt.Errorf("%w: empty host", ErrInvalidURL)
}
portStr := parsedURL.Port()
if portStr == "" {
if hasMalformedWebURLPort(parsedURL.Host) {
return "", fmt.Errorf("%w: invalid port", ErrInvalidURL)
}
} else {
port, err := strconv.Atoi(portStr)
if err != nil || port < 1 || port > 65535 {
return "", fmt.Errorf("%w: invalid port %q", ErrInvalidURL, portStr)
}
}
parsedURL.Host = normalizeWebURLHost(parsedURL.Host)
return fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host), nil
@@ -152,6 +167,14 @@ func normalizeWebURLHost(host string) string {
return host
}
func hasMalformedWebURLPort(host string) bool {
if strings.HasPrefix(host, "[") {
end := strings.LastIndexByte(host, ']')
return end >= 0 && len(host) > end+1 && host[end+1] == ':'
}
return strings.Contains(host, ":")
}
// scanByFingerprints 根据指纹执行POC
func scanByFingerprints(ctx context.Context, target string, fingerprints []string, cfg *common.Config, session *common.ScanSession) {
for _, fingerprint := range fingerprints {
webscan/web_scan_test.go
+75
View File
@@ -1,9 +1,11 @@
package WebScan
import (
"context"
"testing"
"github.com/shadow1ng/fscan/common"
"github.com/shadow1ng/fscan/common/config"
"github.com/shadow1ng/fscan/webscan/lib"
)
@@ -154,6 +156,41 @@ func TestBuildTargetURL(t *testing.T) {
expected: "http://[2001:db8::1]",
expectError: false,
},
{
name: "empty host is rejected",
hostInfo: &common.HostInfo{
Port: 80,
URL: "http://",
},
expectError: true,
},
{
name: "invalid port is rejected",
hostInfo: &common.HostInfo{
Host: "example.com",
Port: 80,
URL: "http://example.com:bad",
},
expectError: true,
},
{
name: "empty explicit port is rejected",
hostInfo: &common.HostInfo{
Host: "example.com",
Port: 80,
URL: "http://example.com:",
},
expectError: true,
},
{
name: "out of range port is rejected",
hostInfo: &common.HostInfo{
Host: "example.com",
Port: 80,
URL: "http://example.com:70000",
},
expectError: true,
},
}
for _, tt := range tests {
@@ -430,6 +467,44 @@ func TestFilterPocsNilSafety(t *testing.T) {
}
}
func TestCreateBaseRequestHeaders(t *testing.T) {
cfg := common.NewConfig()
cfg.HTTP.UserAgent = "fscan-test-agent"
cfg.HTTP.Accept = "application/json"
cfg.HTTP.Cookie = "sid=abc"
req, err := createBaseRequest(context.Background(), "http://example.com/path", cfg)
if err != nil {
t.Fatalf("createBaseRequest error = %v", err)
}
if req.Method != "GET" {
t.Fatalf("method = %q, want GET", req.Method)
}
if got := req.Header.Get("User-agent"); got != "fscan-test-agent" {
t.Fatalf("User-agent = %q", got)
}
if got := req.Header.Get("Accept"); got != "application/json" {
t.Fatalf("Accept = %q", got)
}
if got := req.Header.Get("Cookie"); got != "sid=abc" {
t.Fatalf("Cookie = %q", got)
}
if got := req.Header.Get("Accept-Language"); got == "" {
t.Fatal("Accept-Language should be set")
}
}
func TestExecutePOCsEarlyReturns(t *testing.T) {
cfg := common.NewConfig()
session := common.NewScanSession(cfg, common.NewState(), &common.FlagVars{})
previous := allPocs
allPocs = nil
t.Cleanup(func() { allPocs = previous })
executePOCs(context.Background(), config.PocInfo{}, cfg, session)
executePOCs(context.Background(), config.PocInfo{Target: "http://example.com", PocName: "missing"}, cfg, session)
}
func TestDirectoryExists(t *testing.T) {
tests := []struct {
name string