archive/fscan
1
0
Fork 0
mirror of https://github.com/shadow1ng/fscan.git synced 2026-09-24 04:01:52 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

commit message

Browse Source
This commit is contained in:
shadow1ng
2020-12-29 17:17:10 +08:00
commit df45b07ce8
97 changed files with 45329 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
WebScan/lib/check.go
+222
View File
@@ -0,0 +1,222 @@
package lib
import (
"embed"
"fmt"
"github.com/shadow1ng/fscan/common"
"math/rand"
"net/http"
"net/url"
"regexp"
"sort"
"strings"
"sync"
"time"
)
var (
ceyeApi = "a78a1cb49d91fe09e01876078d1868b2"
ceyeDomain = "7wtusr.ceye.io"
)
type Task struct {
Req *http.Request
Poc *Poc
}
func checkVul(tasks []Task, ticker *time.Ticker) <-chan Task {
var wg sync.WaitGroup
results := make(chan Task)
for _, task := range tasks {
wg.Add(1)
go func(task Task) {
defer wg.Done()
<-ticker.C
isVul, err := executePoc(task.Req, task.Poc)
if err != nil {
return
}
if isVul {
results <- task
}
}(task)
}
go func() {
wg.Wait()
close(results)
}()
return results
}
func CheckMultiPoc(req *http.Request, Pocs embed.FS, rate int,pocname string) {
rateLimit := time.Second / time.Duration(rate)
ticker := time.NewTicker(rateLimit)
defer ticker.Stop()
var tasks []Task
for _, poc := range LoadMultiPoc(Pocs,pocname) {
task := Task{
Req: req,
Poc: poc,
}
tasks = append(tasks, task)
}
for result := range checkVul(tasks, ticker) {
result := fmt.Sprintf("%s %s", result.Req.URL, result.Poc.Name)
common.LogSuccess(result)
}
}
func executePoc(oReq *http.Request, p *Poc) (bool, error) {
c := NewEnvOption()
c.UpdateCompileOptions(p.Set)
env, err := NewEnv(&c)
if err != nil {
fmt.Println("environment creation error: %s\n", err)
return false, err
}
variableMap := make(map[string]interface{})
req, err := ParseRequest(oReq)
if err != nil {
//fmt.Println(err)
return false, err
}
variableMap["request"] = req
// 现在假定set中payload作为最后产出，那么先排序解析其他的自定义变量，更新map[string]interface{}后再来解析payload
keys := make([]string, 0)
for k := range p.Set {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
expression := p.Set[k]
if k != "payload" {
if expression == "newReverse()" {
variableMap[k] = newReverse()
continue
}
out, err := Evaluate(env, expression, variableMap)
if err != nil {
//fmt.Println(err)
continue
}
switch value := out.Value().(type) {
case *UrlType:
variableMap[k] = UrlTypeToString(value)
case int64:
variableMap[k] = int(value)
default:
variableMap[k] = fmt.Sprintf("%v", out)
}
}
}
if p.Set["payload"] != "" {
out, err := Evaluate(env, p.Set["payload"], variableMap)
if err != nil {
return false, err
}
variableMap["payload"] = fmt.Sprintf("%v", out)
}
success := false
for _, rule := range p.Rules {
for k1, v1 := range variableMap {
_, isMap := v1.(map[string]string)
if isMap {
continue
}
value := fmt.Sprintf("%v", v1)
for k2, v2 := range rule.Headers {
rule.Headers[k2] = strings.ReplaceAll(v2, "{{"+k1+"}}", value)
}
rule.Path = strings.ReplaceAll(strings.TrimSpace(rule.Path), "{{"+k1+"}}", value)
rule.Body = strings.ReplaceAll(strings.TrimSpace(rule.Body), "{{"+k1+"}}", value)
}
if oReq.URL.Path != "" && oReq.URL.Path != "/" {
req.Url.Path = fmt.Sprint(oReq.URL.Path, rule.Path)
} else {
req.Url.Path = rule.Path
}
// 某些poc没有区分path和query，需要处理
req.Url.Path = strings.ReplaceAll(req.Url.Path, " ", "%20")
req.Url.Path = strings.ReplaceAll(req.Url.Path, "+", "%20")
newRequest, _ := http.NewRequest(rule.Method, fmt.Sprintf("%s://%s%s", req.Url.Scheme, req.Url.Host, req.Url.Path), strings.NewReader(rule.Body))
newRequest.Header = oReq.Header.Clone()
for k, v := range rule.Headers {
newRequest.Header.Set(k, v)
}
resp, err := DoRequest(newRequest, rule.FollowRedirects)
if err != nil {
return false, err
}
variableMap["response"] = resp
// 先判断响应页面是否匹配search规则
if rule.Search != "" {
result := doSearch(strings.TrimSpace(rule.Search), string(resp.Body))
if result != nil && len(result) > 0 { // 正则匹配成功
for k, v := range result {
variableMap[k] = v
}
//return false, nil
} else {
return false, nil
}
}
out, err := Evaluate(env, rule.Expression, variableMap)
if err != nil {
return false, err
}
//fmt.Println(fmt.Sprintf("%v, %s", out, out.Type().TypeName()))
if fmt.Sprintf("%v", out) == "false" { //如果false不继续执行后续rule
success = false // 如果最后一步执行失败，就算前面成功了最终依旧是失败
break
}
success = true
}
return success, nil
}
func doSearch(re string, body string) map[string]string {
r, err := regexp.Compile(re)
if err != nil {
return nil
}
result := r.FindStringSubmatch(body)
names := r.SubexpNames()
if len(result) > 1 && len(names) > 1 {
paramsMap := make(map[string]string)
for i, name := range names {
if i > 0 && i <= len(result) {
paramsMap[name] = result[i]
}
}
return paramsMap
}
return nil
}
func newReverse() *Reverse {
letters := "1234567890abcdefghijklmnopqrstuvwxyz"
randSource := rand.New(rand.NewSource(time.Now().Unix()))
sub := RandomStr(randSource, letters, 8)
if ceyeDomain == "" {
return &Reverse{}
}
urlStr := fmt.Sprintf("http://%s.%s", sub, ceyeDomain)
u, _ := url.Parse(urlStr)
return &Reverse{
Url: ParseUrl(u),
Domain: u.Hostname(),
Ip: "",
IsDomainNameServer: false,
}
}
WebScan/lib/eval.go
+469
View File
@@ -0,0 +1,469 @@
package lib
import (
"bytes"
"crypto/md5"
"encoding/base64"
"fmt"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/checker/decls"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"github.com/google/cel-go/interpreter/functions"
exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
"math/rand"
"net/http"
"net/url"
"regexp"
"strings"
"time"
)
func NewEnv(c *CustomLib) (*cel.Env, error) {
return cel.NewEnv(cel.Lib(c))
}
func Evaluate(env *cel.Env, expression string, params map[string]interface{}) (ref.Val, error) {
ast, iss := env.Compile(expression)
if iss.Err() != nil {
//fmt.Println("compile: ", iss.Err())
return nil, iss.Err()
}
prg, err := env.Program(ast)
if err != nil {
//fmt.Println("Program creation error: %v", err)
return nil, err
}
out, _, err := prg.Eval(params)
if err != nil {
//fmt.Println("Evaluation error: %v", err)
return nil, err
}
return out, nil
}
func UrlTypeToString(u *UrlType) string {
var buf strings.Builder
if u.Scheme != "" {
buf.WriteString(u.Scheme)
buf.WriteByte(':')
}
if u.Scheme != "" || u.Host != "" {
if u.Host != "" || u.Path != "" {
buf.WriteString("//")
}
if h := u.Host; h != "" {
buf.WriteString(u.Host)
}
}
path := u.Path
if path != "" && path[0] != '/' && u.Host != "" {
buf.WriteByte('/')
}
if buf.Len() == 0 {
if i := strings.IndexByte(path, ':'); i > -1 && strings.IndexByte(path[:i], '/') == -1 {
buf.WriteString("./")
}
}
buf.WriteString(path)
if u.Query != "" {
buf.WriteByte('?')
buf.WriteString(u.Query)
}
if u.Fragment != "" {
buf.WriteByte('#')
buf.WriteString(u.Fragment)
}
return buf.String()
}
type CustomLib struct {
envOptions []cel.EnvOption
programOptions []cel.ProgramOption
}
func NewEnvOption() CustomLib {
c := CustomLib{}
c.envOptions = []cel.EnvOption{
cel.Container("lib"),
cel.Types(
&UrlType{},
&Request{},
&Response{},
&Reverse{},
),
cel.Declarations(
decls.NewIdent("request", decls.NewObjectType("lib.Request"), nil),
decls.NewIdent("response", decls.NewObjectType("lib.Response"), nil),
//decls.NewIdent("reverse", decls.NewObjectType("lib.Reverse"), nil),
),
cel.Declarations(
// functions
decls.NewFunction("bcontains",
decls.NewInstanceOverload("bytes_bcontains_bytes",
[]*exprpb.Type{decls.Bytes, decls.Bytes},
decls.Bool)),
decls.NewFunction("bmatches",
decls.NewInstanceOverload("string_bmatches_bytes",
[]*exprpb.Type{decls.String, decls.Bytes},
decls.Bool)),
decls.NewFunction("md5",
decls.NewOverload("md5_string",
[]*exprpb.Type{decls.String},
decls.String)),
decls.NewFunction("randomInt",
decls.NewOverload("randomInt_int_int",
[]*exprpb.Type{decls.Int, decls.Int},
decls.Int)),
decls.NewFunction("randomLowercase",
decls.NewOverload("randomLowercase_int",
[]*exprpb.Type{decls.Int},
decls.String)),
decls.NewFunction("base64",
decls.NewOverload("base64_string",
[]*exprpb.Type{decls.String},
decls.String)),
decls.NewFunction("base64",
decls.NewOverload("base64_bytes",
[]*exprpb.Type{decls.Bytes},
decls.String)),
decls.NewFunction("base64Decode",
decls.NewOverload("base64Decode_string",
[]*exprpb.Type{decls.String},
decls.String)),
decls.NewFunction("base64Decode",
decls.NewOverload("base64Decode_bytes",
[]*exprpb.Type{decls.Bytes},
decls.String)),
decls.NewFunction("urlencode",
decls.NewOverload("urlencode_string",
[]*exprpb.Type{decls.String},
decls.String)),
decls.NewFunction("urlencode",
decls.NewOverload("urlencode_bytes",
[]*exprpb.Type{decls.Bytes},
decls.String)),
decls.NewFunction("urldecode",
decls.NewOverload("urldecode_string",
[]*exprpb.Type{decls.String},
decls.String)),
decls.NewFunction("urldecode",
decls.NewOverload("urldecode_bytes",
[]*exprpb.Type{decls.Bytes},
decls.String)),
decls.NewFunction("substr",
decls.NewOverload("substr_string_int_int",
[]*exprpb.Type{decls.String, decls.Int, decls.Int},
decls.String)),
decls.NewFunction("wait",
decls.NewInstanceOverload("reverse_wait_int",
[]*exprpb.Type{decls.Any, decls.Int},
decls.Bool)),
decls.NewFunction("icontains",
decls.NewInstanceOverload("icontains_string",
[]*exprpb.Type{decls.String, decls.String},
decls.Bool)),
),
}
c.programOptions = []cel.ProgramOption{
cel.Functions(
&functions.Overload{
Operator: "bytes_bcontains_bytes",
Binary: func(lhs ref.Val, rhs ref.Val) ref.Val {
v1, ok := lhs.(types.Bytes)
if !ok {
return types.ValOrErr(lhs, "unexpected type '%v' passed to bcontains", lhs.Type())
}
v2, ok := rhs.(types.Bytes)
if !ok {
return types.ValOrErr(rhs, "unexpected type '%v' passed to bcontains", rhs.Type())
}
return types.Bool(bytes.Contains(v1, v2))
},
},
&functions.Overload{
Operator: "string_bmatch_bytes",
Binary: func(lhs ref.Val, rhs ref.Val) ref.Val {
v1, ok := lhs.(types.String)
if !ok {
return types.ValOrErr(lhs, "unexpected type '%v' passed to bmatch", lhs.Type())
}
v2, ok := rhs.(types.Bytes)
if !ok {
return types.ValOrErr(rhs, "unexpected type '%v' passed to bmatch", rhs.Type())
}
ok, err := regexp.Match(string(v1), v2)
if err != nil {
return types.NewErr("%v", err)
}
return types.Bool(ok)
},
},
&functions.Overload{
Operator: "md5_string",
Unary: func(value ref.Val) ref.Val {
v, ok := value.(types.String)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to md5_string", value.Type())
}
return types.String(fmt.Sprintf("%x", md5.Sum([]byte(v))))
},
},
&functions.Overload{
Operator: "randomInt_int_int",
Binary: func(lhs ref.Val, rhs ref.Val) ref.Val {
from, ok := lhs.(types.Int)
if !ok {
return types.ValOrErr(lhs, "unexpected type '%v' passed to randomInt", lhs.Type())
}
to, ok := rhs.(types.Int)
if !ok {
return types.ValOrErr(rhs, "unexpected type '%v' passed to randomInt", rhs.Type())
}
min, max := int(from), int(to)
return types.Int(rand.Intn(max-min) + min)
},
},
&functions.Overload{
Operator: "randomLowercase_int",
Unary: func(value ref.Val) ref.Val {
n, ok := value.(types.Int)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to randomLowercase", value.Type())
}
return types.String(randomLowercase(int(n)))
},
},
&functions.Overload{
Operator: "base64_string",
Unary: func(value ref.Val) ref.Val {
v, ok := value.(types.String)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to base64_string", value.Type())
}
return types.String(base64.StdEncoding.EncodeToString([]byte(v)))
},
},
&functions.Overload{
Operator: "base64_bytes",
Unary: func(value ref.Val) ref.Val {
v, ok := value.(types.Bytes)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to base64_bytes", value.Type())
}
return types.String(base64.StdEncoding.EncodeToString(v))
},
},
&functions.Overload{
Operator: "base64Decode_string",
Unary: func(value ref.Val) ref.Val {
v, ok := value.(types.String)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to base64Decode_string", value.Type())
}
decodeBytes, err := base64.StdEncoding.DecodeString(string(v))
if err != nil {
return types.NewErr("%v", err)
}
return types.String(decodeBytes)
},
},
&functions.Overload{
Operator: "base64Decode_bytes",
Unary: func(value ref.Val) ref.Val {
v, ok := value.(types.Bytes)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to base64Decode_bytes", value.Type())
}
decodeBytes, err := base64.StdEncoding.DecodeString(string(v))
if err != nil {
return types.NewErr("%v", err)
}
return types.String(decodeBytes)
},
},
&functions.Overload{
Operator: "urlencode_string",
Unary: func(value ref.Val) ref.Val {
v, ok := value.(types.String)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to urlencode_string", value.Type())
}
return types.String(url.QueryEscape(string(v)))
},
},
&functions.Overload{
Operator: "urlencode_bytes",
Unary: func(value ref.Val) ref.Val {
v, ok := value.(types.Bytes)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to urlencode_bytes", value.Type())
}
return types.String(url.QueryEscape(string(v)))
},
},
&functions.Overload{
Operator: "urldecode_string",
Unary: func(value ref.Val) ref.Val {
v, ok := value.(types.String)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to urldecode_string", value.Type())
}
decodeString, err := url.QueryUnescape(string(v))
if err != nil {
return types.NewErr("%v", err)
}
return types.String(decodeString)
},
},
&functions.Overload{
Operator: "urldecode_bytes",
Unary: func(value ref.Val) ref.Val {
v, ok := value.(types.Bytes)
if !ok {
return types.ValOrErr(value, "unexpected type '%v' passed to urldecode_bytes", value.Type())
}
decodeString, err := url.QueryUnescape(string(v))
if err != nil {
return types.NewErr("%v", err)
}
return types.String(decodeString)
},
},
&functions.Overload{
Operator: "substr_string_int_int",
Function: func(values ...ref.Val) ref.Val {
if len(values) == 3 {
str, ok := values[0].(types.String)
if !ok {
return types.NewErr("invalid string to 'substr'")
}
start, ok := values[1].(types.Int)
if !ok {
return types.NewErr("invalid start to 'substr'")
}
length, ok := values[2].(types.Int)
if !ok {
return types.NewErr("invalid length to 'substr'")
}
runes := []rune(str)
if start < 0 || length < 0 || int(start+length) > len(runes) {
return types.NewErr("invalid start or length to 'substr'")
}
return types.String(runes[start : start+length])
} else {
return types.NewErr("too many arguments to 'substr'")
}
},
},
&functions.Overload{
Operator: "reverse_wait_int",
Binary: func(lhs ref.Val, rhs ref.Val) ref.Val {
reverse, ok := lhs.Value().(*Reverse)
if !ok {
return types.ValOrErr(lhs, "unexpected type '%v' passed to 'wait'", lhs.Type())
}
timeout, ok := rhs.Value().(int64)
if !ok {
return types.ValOrErr(rhs, "unexpected type '%v' passed to 'wait'", rhs.Type())
}
return types.Bool(reverseCheck(reverse, timeout))
},
},
&functions.Overload{
Operator: "icontains_string",
Binary: func(lhs ref.Val, rhs ref.Val) ref.Val {
v1, ok := lhs.(types.String)
if !ok {
return types.ValOrErr(lhs, "unexpected type '%v' passed to bcontains", lhs.Type())
}
v2, ok := rhs.(types.String)
if !ok {
return types.ValOrErr(rhs, "unexpected type '%v' passed to bcontains", rhs.Type())
}
// 不区分大小写包含
return types.Bool(strings.Contains(strings.ToLower(string(v1)), strings.ToLower(string(v2))))
},
},
),
}
return c
}
// 声明环境中的变量类型和函数
func (c *CustomLib) CompileOptions() []cel.EnvOption {
return c.envOptions
}
func (c *CustomLib) ProgramOptions() []cel.ProgramOption {
return c.programOptions
}
func (c *CustomLib) UpdateCompileOptions(args map[string]string) {
for k, v := range args {
// 在执行之前是不知道变量的类型的，所以统一声明为字符型
// 所以randomInt虽然返回的是int型，在运算中却被当作字符型进行计算，需要重载string_*_string
var d *exprpb.Decl
if strings.HasPrefix(v, "randomInt") {
d = decls.NewIdent(k, decls.Int, nil)
} else if strings.HasPrefix(v, "newReverse") {
d = decls.NewIdent(k, decls.NewObjectType("lib.Reverse"), nil)
} else {
d = decls.NewIdent(k, decls.String, nil)
}
c.envOptions = append(c.envOptions, cel.Declarations(d))
}
}
func randomLowercase(n int) string {
lowercase := "abcdefghijklmnopqrstuvwxyz"
randSource := rand.New(rand.NewSource(time.Now().Unix()))
return RandomStr(randSource, lowercase, n)
}
func reverseCheck(r *Reverse, timeout int64) bool {
if ceyeApi == "" || r.Domain == "" {
return false
}
time.Sleep(time.Second * time.Duration(timeout))
sub := strings.Split(r.Domain, ".")[0]
urlStr := fmt.Sprintf("http://api.ceye.io/v1/records?token=%s&type=dns&filter=%s", ceyeApi, sub)
fmt.Println(urlStr)
req, _ := http.NewRequest("GET", urlStr, nil)
resp, err := DoRequest(req, false)
if err != nil {
return false
}
if !bytes.Contains(resp.Body, []byte(`"data": []`)) && bytes.Contains(resp.Body, []byte(`"message": "OK"`)) { // api返回结果不为空
return true
}
return false
}
func RandomStr(randSource *rand.Rand, letterBytes string, n int) string {
const (
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
//letterBytes = "1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
)
randBytes := make([]byte, n)
for i, cache, remain := n-1, randSource.Int63(), letterIdxMax; i >= 0; {
if remain == 0 {
cache, remain = randSource.Int63(), letterIdxMax
}
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
randBytes[i] = letterBytes[idx]
i--
}
cache >>= letterIdxBits
remain--
}
return string(randBytes)
}
WebScan/lib/http.go
+171
View File
@@ -0,0 +1,171 @@
package lib
import (
"bytes"
"compress/gzip"
"crypto/tls"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"strconv"
"time"
)
var (
client *http.Client
clientNoRedirect *http.Client
dialTimout = 5 * time.Second
keepAlive = 15 * time.Second
)
func InitHttpClient(ThreadsNum int, DownProxy string, Timeout time.Duration) error {
dialer := &net.Dialer{
Timeout: dialTimout,
KeepAlive: keepAlive,
}
tr := &http.Transport{
DialContext: dialer.DialContext,
//MaxConnsPerHost: 0,
MaxIdleConns: 1000,
MaxIdleConnsPerHost: ThreadsNum * 2,
IdleConnTimeout: keepAlive,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
TLSHandshakeTimeout: 5 * time.Second,
DisableKeepAlives: false,
}
if DownProxy != "" {
u, err := url.Parse(DownProxy)
if err != nil {
return err
}
tr.Proxy = http.ProxyURL(u)
}
client = &http.Client{
Transport: tr,
Timeout: Timeout,
}
clientNoRedirect = &http.Client{
Transport: tr,
Timeout: Timeout,
}
clientNoRedirect.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
return nil
}
func DoRequest(req *http.Request, redirect bool) (*Response, error) {
if req.Body == nil || req.Body == http.NoBody {
} else {
req.Header.Set("Content-Length", strconv.Itoa(int(req.ContentLength)))
if req.Header.Get("Content-Type") == "" {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
}
var oResp *http.Response
var err error
if redirect {
oResp, err = client.Do(req)
} else {
oResp, err = clientNoRedirect.Do(req)
}
if err != nil {
return nil, err
}
defer oResp.Body.Close()
resp, err := ParseResponse(oResp)
if err != nil {
return nil, err
}
return resp, err
}
func ParseUrl(u *url.URL) *UrlType {
nu := &UrlType{}
nu.Scheme = u.Scheme
nu.Domain = u.Hostname()
nu.Host = u.Host
nu.Port = u.Port()
nu.Path = u.EscapedPath()
nu.Query = u.RawQuery
nu.Fragment = u.Fragment
return nu
}
func ParseRequest(oReq *http.Request) (*Request, error) {
req := &Request{}
req.Method = oReq.Method
req.Url = ParseUrl(oReq.URL)
header := make(map[string]string)
for k := range oReq.Header {
header[k] = oReq.Header.Get(k)
}
req.Headers = header
req.ContentType = oReq.Header.Get("Content-Type")
if oReq.Body == nil || oReq.Body == http.NoBody {
} else {
data, err := ioutil.ReadAll(oReq.Body)
if err != nil {
return nil, err
}
req.Body = data
oReq.Body = ioutil.NopCloser(bytes.NewBuffer(data))
}
return req, nil
}
func ParseResponse(oResp *http.Response) (*Response, error) {
var resp Response
header := make(map[string]string)
resp.Status = int32(oResp.StatusCode)
resp.Url = ParseUrl(oResp.Request.URL)
for k := range oResp.Header {
header[k] = oResp.Header.Get(k)
}
resp.Headers = header
resp.ContentType = oResp.Header.Get("Content-Type")
body, err := getRespBody(oResp)
if err != nil {
return nil, err
}
resp.Body = body
return &resp, nil
}
func getRespBody(oResp *http.Response) ([]byte, error) {
var body []byte
if oResp.Header.Get("Content-Encoding") == "gzip" {
gr, err := gzip.NewReader(oResp.Body)
if err != nil {
return nil, err
}
defer gr.Close()
for {
buf := make([]byte, 1024)
n, err := gr.Read(buf)
if err != nil && err != io.EOF {
//utils.Logger.Error(err)
return nil, err
}
if n == 0 {
break
}
body = append(body, buf...)
}
} else {
raw, err := ioutil.ReadAll(oResp.Body)
if err != nil {
//utils.Logger.Error(err)
return nil, err
}
defer oResp.Body.Close()
body = raw
}
return body, nil
}
WebScan/lib/http.pb.go
+354
View File
@@ -0,0 +1,354 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: http.proto
package lib
import (
fmt "fmt"
proto "github.com/golang/protobuf/proto"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
type UrlType struct {
Scheme string `protobuf:"bytes,1,opt,name=scheme,proto3" json:"scheme,omitempty"`
Domain string `protobuf:"bytes,2,opt,name=domain,proto3" json:"domain,omitempty"`
Host string `protobuf:"bytes,3,opt,name=host,proto3" json:"host,omitempty"`
Port string `protobuf:"bytes,4,opt,name=port,proto3" json:"port,omitempty"`
Path string `protobuf:"bytes,5,opt,name=path,proto3" json:"path,omitempty"`
Query string `protobuf:"bytes,6,opt,name=query,proto3" json:"query,omitempty"`
Fragment string `protobuf:"bytes,7,opt,name=fragment,proto3" json:"fragment,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *UrlType) Reset() { *m = UrlType{} }
func (m *UrlType) String() string { return proto.CompactTextString(m) }
func (*UrlType) ProtoMessage() {}
func (*UrlType) Descriptor() ([]byte, []int) {
return fileDescriptor_11b04836674e6f94, []int{0}
}
func (m *UrlType) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_UrlType.Unmarshal(m, b)
}
func (m *UrlType) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_UrlType.Marshal(b, m, deterministic)
}
func (m *UrlType) XXX_Merge(src proto.Message) {
xxx_messageInfo_UrlType.Merge(m, src)
}
func (m *UrlType) XXX_Size() int {
return xxx_messageInfo_UrlType.Size(m)
}
func (m *UrlType) XXX_DiscardUnknown() {
xxx_messageInfo_UrlType.DiscardUnknown(m)
}
var xxx_messageInfo_UrlType proto.InternalMessageInfo
func (m *UrlType) GetScheme() string {
if m != nil {
return m.Scheme
}
return ""
}
func (m *UrlType) GetDomain() string {
if m != nil {
return m.Domain
}
return ""
}
func (m *UrlType) GetHost() string {
if m != nil {
return m.Host
}
return ""
}
func (m *UrlType) GetPort() string {
if m != nil {
return m.Port
}
return ""
}
func (m *UrlType) GetPath() string {
if m != nil {
return m.Path
}
return ""
}
func (m *UrlType) GetQuery() string {
if m != nil {
return m.Query
}
return ""
}
func (m *UrlType) GetFragment() string {
if m != nil {
return m.Fragment
}
return ""
}
type Request struct {
Url *UrlType `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
Method string `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"`
Headers map[string]string `protobuf:"bytes,3,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
ContentType string `protobuf:"bytes,4,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"`
Body []byte `protobuf:"bytes,5,opt,name=body,proto3" json:"body,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Request) Reset() { *m = Request{} }
func (m *Request) String() string { return proto.CompactTextString(m) }
func (*Request) ProtoMessage() {}
func (*Request) Descriptor() ([]byte, []int) {
return fileDescriptor_11b04836674e6f94, []int{1}
}
func (m *Request) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Request.Unmarshal(m, b)
}
func (m *Request) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Request.Marshal(b, m, deterministic)
}
func (m *Request) XXX_Merge(src proto.Message) {
xxx_messageInfo_Request.Merge(m, src)
}
func (m *Request) XXX_Size() int {
return xxx_messageInfo_Request.Size(m)
}
func (m *Request) XXX_DiscardUnknown() {
xxx_messageInfo_Request.DiscardUnknown(m)
}
var xxx_messageInfo_Request proto.InternalMessageInfo
func (m *Request) GetUrl() *UrlType {
if m != nil {
return m.Url
}
return nil
}
func (m *Request) GetMethod() string {
if m != nil {
return m.Method
}
return ""
}
func (m *Request) GetHeaders() map[string]string {
if m != nil {
return m.Headers
}
return nil
}
func (m *Request) GetContentType() string {
if m != nil {
return m.ContentType
}
return ""
}
func (m *Request) GetBody() []byte {
if m != nil {
return m.Body
}
return nil
}
type Response struct {
Url *UrlType `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
Status int32 `protobuf:"varint,2,opt,name=status,proto3" json:"status,omitempty"`
Headers map[string]string `protobuf:"bytes,3,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
ContentType string `protobuf:"bytes,4,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"`
Body []byte `protobuf:"bytes,5,opt,name=body,proto3" json:"body,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Response) Reset() { *m = Response{} }
func (m *Response) String() string { return proto.CompactTextString(m) }
func (*Response) ProtoMessage() {}
func (*Response) Descriptor() ([]byte, []int) {
return fileDescriptor_11b04836674e6f94, []int{2}
}
func (m *Response) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Response.Unmarshal(m, b)
}
func (m *Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Response.Marshal(b, m, deterministic)
}
func (m *Response) XXX_Merge(src proto.Message) {
xxx_messageInfo_Response.Merge(m, src)
}
func (m *Response) XXX_Size() int {
return xxx_messageInfo_Response.Size(m)
}
func (m *Response) XXX_DiscardUnknown() {
xxx_messageInfo_Response.DiscardUnknown(m)
}
var xxx_messageInfo_Response proto.InternalMessageInfo
func (m *Response) GetUrl() *UrlType {
if m != nil {
return m.Url
}
return nil
}
func (m *Response) GetStatus() int32 {
if m != nil {
return m.Status
}
return 0
}
func (m *Response) GetHeaders() map[string]string {
if m != nil {
return m.Headers
}
return nil
}
func (m *Response) GetContentType() string {
if m != nil {
return m.ContentType
}
return ""
}
func (m *Response) GetBody() []byte {
if m != nil {
return m.Body
}
return nil
}
type Reverse struct {
Url *UrlType `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
Domain string `protobuf:"bytes,2,opt,name=domain,proto3" json:"domain,omitempty"`
Ip string `protobuf:"bytes,3,opt,name=ip,proto3" json:"ip,omitempty"`
IsDomainNameServer bool `protobuf:"varint,4,opt,name=is_domain_name_server,json=isDomainNameServer,proto3" json:"is_domain_name_server,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Reverse) Reset() { *m = Reverse{} }
func (m *Reverse) String() string { return proto.CompactTextString(m) }
func (*Reverse) ProtoMessage() {}
func (*Reverse) Descriptor() ([]byte, []int) {
return fileDescriptor_11b04836674e6f94, []int{3}
}
func (m *Reverse) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Reverse.Unmarshal(m, b)
}
func (m *Reverse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Reverse.Marshal(b, m, deterministic)
}
func (m *Reverse) XXX_Merge(src proto.Message) {
xxx_messageInfo_Reverse.Merge(m, src)
}
func (m *Reverse) XXX_Size() int {
return xxx_messageInfo_Reverse.Size(m)
}
func (m *Reverse) XXX_DiscardUnknown() {
xxx_messageInfo_Reverse.DiscardUnknown(m)
}
var xxx_messageInfo_Reverse proto.InternalMessageInfo
func (m *Reverse) GetUrl() *UrlType {
if m != nil {
return m.Url
}
return nil
}
func (m *Reverse) GetDomain() string {
if m != nil {
return m.Domain
}
return ""
}
func (m *Reverse) GetIp() string {
if m != nil {
return m.Ip
}
return ""
}
func (m *Reverse) GetIsDomainNameServer() bool {
if m != nil {
return m.IsDomainNameServer
}
return false
}
func init() {
proto.RegisterType((*UrlType)(nil), "lib.UrlType")
proto.RegisterType((*Request)(nil), "lib.Request")
proto.RegisterMapType((map[string]string)(nil), "lib.Request.HeadersEntry")
proto.RegisterType((*Response)(nil), "lib.Response")
proto.RegisterMapType((map[string]string)(nil), "lib.Response.HeadersEntry")
proto.RegisterType((*Reverse)(nil), "lib.Reverse")
}
func init() {
proto.RegisterFile("http.proto", fileDescriptor_11b04836674e6f94)
}
var fileDescriptor_11b04836674e6f94 = []byte{
// 378 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x93, 0xb1, 0x8e, 0xd3, 0x40,
0x10, 0x86, 0x65, 0x3b, 0x89, 0xc3, 0xc4, 0x42, 0x68, 0x05, 0x68, 0x49, 0x81, 0x8e, 0x54, 0x57,
0x59, 0xe2, 0x8e, 0x02, 0x5d, 0x0d, 0x12, 0x15, 0xc5, 0x02, 0xb5, 0xb5, 0x3e, 0x0f, 0xd8, 0xc2,
0xf6, 0x6e, 0x76, 0xc7, 0x91, 0xdc, 0xf3, 0x2e, 0x3c, 0x1b, 0xe2, 0x25, 0x90, 0x67, 0x37, 0x08,
0x21, 0x8a, 0x94, 0x74, 0xf3, 0xff, 0xbf, 0x3d, 0x9a, 0x6f, 0x3c, 0x06, 0x68, 0x89, 0x6c, 0x69,
0x9d, 0x21, 0x23, 0xb2, 0xbe, 0xab, 0x0f, 0xdf, 0x13, 0xc8, 0x3f, 0xb9, 0xfe, 0xe3, 0x6c, 0x51,
0x3c, 0x85, 0x8d, 0xbf, 0x6f, 0x71, 0x40, 0x99, 0x5c, 0x25, 0xd7, 0x0f, 0x54, 0x54, 0x8b, 0xdf,
0x98, 0x41, 0x77, 0xa3, 0x4c, 0x83, 0x1f, 0x94, 0x10, 0xb0, 0x6a, 0x8d, 0x27, 0x99, 0xb1, 0xcb,
0xf5, 0xe2, 0x59, 0xe3, 0x48, 0xae, 0x82, 0xb7, 0xd4, 0xec, 0x69, 0x6a, 0xe5, 0x3a, 0x7a, 0x9a,
0x5a, 0xf1, 0x18, 0xd6, 0xc7, 0x09, 0xdd, 0x2c, 0x37, 0x6c, 0x06, 0x21, 0xf6, 0xb0, 0xfd, 0xec,
0xf4, 0x97, 0x01, 0x47, 0x92, 0x39, 0x07, 0xbf, 0xf5, 0xe1, 0x47, 0x02, 0xb9, 0xc2, 0xe3, 0x84,
0x9e, 0xc4, 0x73, 0xc8, 0x26, 0xd7, 0xf3, 0x98, 0xbb, 0x9b, 0xa2, 0xec, 0xbb, 0xba, 0x8c, 0x10,
0x6a, 0x09, 0x96, 0x89, 0x07, 0xa4, 0xd6, 0x34, 0xe7, 0x89, 0x83, 0x12, 0xb7, 0x90, 0xb7, 0xa8,
0x1b, 0x74, 0x5e, 0x66, 0x57, 0xd9, 0xf5, 0xee, 0xe6, 0x19, 0xbf, 0x1b, 0xdb, 0x96, 0xef, 0x42,
0xf6, 0x76, 0x24, 0x37, 0xab, 0xf3, 0x93, 0xe2, 0x05, 0x14, 0xf7, 0x66, 0x24, 0x1c, 0xa9, 0xa2,
0xd9, 0x62, 0x44, 0xdb, 0x45, 0x8f, 0x37, 0x27, 0x60, 0x55, 0x9b, 0x66, 0x66, 0xc2, 0x42, 0x71,
0xbd, 0xbf, 0x83, 0xe2, 0xcf, 0x7e, 0xe2, 0x11, 0x64, 0x5f, 0x71, 0x8e, 0xab, 0x5d, 0xca, 0x65,
0x07, 0x27, 0xdd, 0x4f, 0x18, 0x87, 0x0c, 0xe2, 0x2e, 0x7d, 0x9d, 0x1c, 0x7e, 0x26, 0xb0, 0x55,
0xe8, 0xad, 0x19, 0x3d, 0x5e, 0x02, 0xeb, 0x49, 0xd3, 0xe4, 0xb9, 0xcf, 0x5a, 0x45, 0x25, 0x5e,
0xfd, 0x0d, 0xbb, 0x8f, 0xb0, 0xa1, 0xef, 0xff, 0x43, 0xfb, 0x8d, 0xbf, 0xec, 0x09, 0xdd, 0x65,
0xb0, 0xff, 0xbc, 0xc5, 0x87, 0x90, 0x76, 0x36, 0x5e, 0x62, 0xda, 0x59, 0xf1, 0x12, 0x9e, 0x74,
0xbe, 0x0a, 0x61, 0x35, 0xea, 0x01, 0x2b, 0x8f, 0xee, 0x84, 0x8e, 0x79, 0xb6, 0x4a, 0x74, 0xfe,
0x0d, 0x67, 0xef, 0xf5, 0x80, 0x1f, 0x38, 0xa9, 0x37, 0xfc, 0x5b, 0xdc, 0xfe, 0x0a, 0x00, 0x00,
0xff, 0xff, 0x2a, 0xe0, 0x6d, 0x45, 0x24, 0x03, 0x00, 0x00,
}
WebScan/lib/http.proto
+35
View File
@@ -0,0 +1,35 @@
syntax = "proto3";
package lib;
message UrlType {
string scheme = 1;
string domain = 2;
string host = 3;
string port = 4;
string path = 5;
string query = 6;
string fragment = 7;
}
message Request {
UrlType url = 1;
string method = 2;
map<string, string> headers = 3;
string content_type = 4;
bytes body = 5;
}
message Response {
UrlType url = 1;
int32 status = 2 ;
map<string, string> headers = 3;
string content_type = 4;
bytes body = 5;
}
message Reverse {
UrlType url = 1;
string domain = 2;
string ip = 3;
bool is_domain_name_server = 4;
}
WebScan/lib/poc.go
+70
View File
@@ -0,0 +1,70 @@
package lib
import (
"embed"
"fmt"
"gopkg.in/yaml.v3"
"strings"
)
type Poc struct {
Name string `yaml:"name"`
Set map[string]string `yaml:"set"`
Rules []Rules `yaml:"rules"`
Detail Detail `yaml:"detail"`
}
type Rules struct {
Method string `yaml:"method"`
Path string `yaml:"path"`
Headers map[string]string `yaml:"headers"`
Body string `yaml:"body"`
Search string `yaml:"search"`
FollowRedirects bool `yaml:"follow_redirects"`
Expression string `yaml:"expression"`
}
type Detail struct {
Author string `yaml:"author"`
Links []string `yaml:"links"`
Description string `yaml:"description"`
Version string `yaml:"version"`
}
func LoadMultiPoc(Pocs embed.FS, pocname string) []*Poc {
var pocs []*Poc
for _, f := range SelectPoc(Pocs, pocname) {
if p, err := loadPoc(f, Pocs); err == nil {
pocs = append(pocs, p)
}
}
return pocs
}
func loadPoc(fileName string, Pocs embed.FS) (*Poc, error) {
p := &Poc{}
yamlFile, err := Pocs.ReadFile("pocs/" + fileName)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(yamlFile, p)
if err != nil {
return nil, err
}
return p, err
}
func SelectPoc(Pocs embed.FS, pocname string) []string {
entries, err := Pocs.ReadDir("pocs")
if err != nil {
fmt.Println(err)
}
var foundFiles []string
for _, entry := range entries {
if strings.Contains(entry.Name(), pocname) {
foundFiles = append(foundFiles, entry.Name())
}
}
return foundFiles
}