From b39ee101f94036a16137ecbd767453241cfd0db8 Mon Sep 17 00:00:00 2001 From: Li4n0 <34324462+Li4n0@users.noreply.github.com> Date: Wed, 5 May 2021 12:59:50 +0800 Subject: [PATCH] feat(ftp): support upload, download and PASV rebind (#13) Co-authored-by: E99p1ant <524306184@qq.com> --- frontend/src/App.vue | 17 +- frontend/src/views/logs/Ftp.vue | 35 +++- frontend/src/views/logs/Mysql.vue | 2 +- frontend/src/views/rules/Ftp.vue | 37 +++- {pkg/mysql => internal/file}/file.go | 44 +++-- internal/rule/compile.go | 16 +- pkg/ftp/ftp.go | 252 ++++++++++++++++++++------- pkg/ftp/record.go | 22 ++- pkg/ftp/rule.go | 2 + pkg/mysql/mysql.go | 38 ++-- pkg/mysql/record.go | 15 +- pkg/mysql/vmysql/conn.go | 18 +- pkg/mysql/vmysql/server.go | 6 +- pkg/server/router.go | 3 +- pkg/server/server.go | 13 +- 15 files changed, 374 insertions(+), 146 deletions(-) rename {pkg/mysql => internal/file}/file.go (61%) diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 5b2d0b1..a5a26a9 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -55,7 +55,7 @@ @click="() => (collapsed = !collapsed)" /> -
+
@@ -111,6 +111,11 @@ export default { openKeys: ['logs', "rules"], }; }, + computed: { + isLogMode() { + return this.$route.path.includes('logs') + } + }, methods: { timing() { if (this.timer !== null) { @@ -122,7 +127,7 @@ export default { } }, mounted() { - if (this.autoRefresh) { + if (this.autoRefresh && this.isLogMode) { this.timing() } }, @@ -138,6 +143,14 @@ export default { } localStorage.setItem('autoRefresh', val) }, + isLogMode(val) { + if (!val) { + clearInterval(this.timer) + } else { + this.timing() + } + localStorage.setItem('autoRefresh', val) + }, refreshInterval(val) { clearInterval(this.timer) this.timing() diff --git a/frontend/src/views/logs/Ftp.vue b/frontend/src/views/logs/Ftp.vue index 25a4e84..cf90ed0 100644 --- a/frontend/src/views/logs/Ftp.vue +++ b/frontend/src/views/logs/Ftp.vue @@ -39,6 +39,18 @@ type="search" :style="{ color: filtered ? '#108ee9' : undefined }" /> + + {{ method }} + + + TRUE + + FALSE + + {{ new Date(time).format("yyyy-MM-dd hh:mm:ss") }} @@ -65,7 +77,7 @@ import FilterDropdown from '@/components/FilterDropdown' const colors = { "CRASHED": "#f50", - "FINISHED": "#87d068" + "FINISHED": "#52c41a" } const columns = [ @@ -128,6 +140,24 @@ const columns = [ filterIcon: 'filterIcon', }, }, + { + title: 'METHOD', + dataIndex: 'method', + key: 'method', + scopedSlots: { + customRender: 'method', + filterDropdown: 'selectDropdown', + filterIcon: 'filterIcon', + }, + }, + { + title: 'FILE', + dataIndex: 'file', + key: 'file', + scopedSlots: { + customRender: 'file' + }, + }, { title: 'STATUS', dataIndex: 'status', @@ -169,6 +199,9 @@ export default { }; }, methods: { + aa(file) { + console.log(file) + }, handleTableChange(pagination, filters, sorter) { const pager = {...this.pagination}; pager.current = pagination.current; diff --git a/frontend/src/views/logs/Mysql.vue b/frontend/src/views/logs/Mysql.vue index fad7463..b00320b 100644 --- a/frontend/src/views/logs/Mysql.vue +++ b/frontend/src/views/logs/Mysql.vue @@ -50,7 +50,7 @@ TrueFalse + >TRUEFALSE Pasv Address + title="1. Support template such as ${user}/${password}/${varname}. +2.For setting the rebind mode, please use ',' to separate addresses."> @@ -33,6 +34,26 @@ + + + + + Data + + + + + + + +
\ No newline at end of file + + \ No newline at end of file diff --git a/pkg/mysql/file.go b/internal/file/file.go similarity index 61% rename from pkg/mysql/file.go rename to internal/file/file.go index 155355e..ca02e31 100644 --- a/pkg/mysql/file.go +++ b/internal/file/file.go @@ -1,16 +1,13 @@ -package mysql +package file import ( "fmt" - "strings" "github.com/gabriel-vasile/mimetype" "github.com/gin-gonic/gin" "github.com/li4n0/revsuit/internal/database" ) -const FILE_SPEARATOR = ";" - type File struct { ID uint `gorm:"primarykey" form:"id" json:"id"` RecordID uint `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;" json:"-"` @@ -18,15 +15,20 @@ type File struct { Content []byte `json:"-"` } -//func (File) TableName() string { -// return "mysql_files" -//} +type MySQLFile File + +func (MySQLFile) TableName() string { + return "mysql_files" +} + +type FTPFile File + +func (FTPFile) TableName() string { + return "ftp_files" +} func GetFile(c *gin.Context) { - var ( - file File - mysqlRecord Record - ) + var file File id := c.Param("id") if id == "" { c.JSON(400, gin.H{ @@ -37,8 +39,21 @@ func GetFile(c *gin.Context) { return } - database.DB.Model(&file).Where("id = ?", id).Find(&file) - database.DB.Model(&mysqlRecord).Where("id = ?", file.RecordID).Find(&mysqlRecord) + recordType := c.Param("record_type") + if id == "" { + c.JSON(400, gin.H{ + "status": "failed", + "error": fmt.Errorf("param record_type missed"), + "result": nil, + }) + return + } + if recordType == "mysql" { + database.DB.Table("mysql_files").Where("id = ?", id).Find(&file) + } else if recordType == "ftp" { + database.DB.Table("ftp_files").Where("id = ?", id).Find(&file) + } + if file.Content == nil || len(file.Content) == 0 { c.JSON(400, gin.H{ "status": "failed", @@ -51,8 +66,7 @@ func GetFile(c *gin.Context) { c.Header("Content-Type", mime.String()) c.Header("Content-Disposition", fmt.Sprintf( - "filename=%s_%s_%d", - strings.Replace(mysqlRecord.RemoteIP, ".", "_", -1), + "filename=%s_%d", file.Name, file.ID, ), diff --git a/internal/rule/compile.go b/internal/rule/compile.go index c6e054a..318304a 100644 --- a/internal/rule/compile.go +++ b/internal/rule/compile.go @@ -1,12 +1,20 @@ package rule -import "strings" +import ( + "strings" +) -func CompileTpl(tpl string, vars map[string]string) (compiled string) { - compiled = tpl +// CompileTpl receive []byte or string type tpl and variables map. +// Return the template after variable substitution +func CompileTpl(tpl interface{}, vars map[string]string) (compiled string) { + switch v := tpl.(type) { + case string: + compiled = v + case []byte: + compiled = string(v) + } for n, v := range vars { compiled = strings.ReplaceAll(compiled, "${"+n+"}", v) } - return compiled } diff --git a/pkg/ftp/ftp.go b/pkg/ftp/ftp.go index 87d9943..438220d 100644 --- a/pkg/ftp/ftp.go +++ b/pkg/ftp/ftp.go @@ -3,6 +3,8 @@ package ftp import ( "bytes" "fmt" + "go/types" + "io" "net" "strconv" "strings" @@ -10,16 +12,19 @@ import ( "time" "github.com/li4n0/revsuit/internal/database" + "github.com/li4n0/revsuit/internal/file" "github.com/li4n0/revsuit/internal/qqwry" "github.com/li4n0/revsuit/internal/recycler" "github.com/li4n0/revsuit/internal/rule" + "github.com/patrickmn/go-cache" log "unknwon.dev/clog/v2" ) type Server struct { Config - rules []*Rule - rulesLock sync.RWMutex + rules []*Rule + rulesLock sync.RWMutex + dataChannel chan map[string]interface{} } type Status string @@ -29,14 +34,22 @@ const ( FINISHED Status = "FINISHED" ) +type Method = string + +const ( + DOWNLOAD Method = "DOWNLOAD" + UPLOAD Method = "UPLOAD" +) + var ( - server *Server - once sync.Once + server *Server + once sync.Once + rebindingCache = cache.New(5*time.Second, 10*time.Second) ) func GetServer() *Server { once.Do(func() { - server = &Server{rulesLock: sync.RWMutex{}} + server = &Server{rulesLock: sync.RWMutex{}, dataChannel: make(chan map[string]interface{}, 10)} }) return server } @@ -54,7 +67,40 @@ func (s *Server) updateRules() error { return db.Order("rank desc").Find(&s.rules).Error } +func (s *Server) authenticate(user, password string) (_rule *Rule, flag, flagGroup string, vars map[string]string) { + for _, _rule := range s.getRules() { + for _, s := range []string{user, password} { + flag, flagGroup, vars = _rule.Match(s) + if flag != "" { + vars["user"] = user + vars["password"] = password + return _rule, flag, flagGroup, vars + } + } + } + return _rule, flag, flagGroup, vars +} + +func (s *Server) getPasvAddressFromCache(ip, pasvAddressTpl string) (pasvAddress string) { + if strings.Contains(pasvAddressTpl, ",") { + values, ok := rebindingCache.Get(ip) + if !ok { + rebindingCache.Set(ip, strings.Split(pasvAddressTpl, ","), cache.DefaultExpiration) + values = strings.Split(pasvAddressTpl, ",") + } + //Choose and delete first address + pasvAddress = values.([]string)[0] + if len(values.([]string)) > 1 { + rebindingCache.Set(ip, values.([]string)[1:len(values.([]string))], cache.DefaultExpiration) + } else { + rebindingCache.Delete(ip) + } + } + return pasvAddress +} + func (s *Server) handleConnection(conn net.Conn) { + log.Trace("New FTP connection from addr [%s]", conn.RemoteAddr()) defer func() { _ = conn.Close() if err := recover(); err != nil { @@ -70,13 +116,18 @@ func (s *Server) handleConnection(conn net.Conn) { log.Warn("FTP write connection error:%v", err) } - ip := strings.Split(conn.RemoteAddr().String(), ":")[0] + ip, port, _ := net.SplitHostPort(conn.RemoteAddr().String()) + dataPort, _ := strconv.Atoi(port) + clientPasvConnAddress := fmt.Sprintf("%s:%d", ip, dataPort+1) buf := &bytes.Buffer{} - var user, password, path, flag, flagGroup string + var user, password, method, flag, flagGroup, pasvAddress, filename string status := CRASHED - var matchedRule *Rule + path := "/" + uploadData := make([]byte, 0) + var _rule *Rule var vars map[string]string + var isRedirect bool loop: for { @@ -88,81 +139,125 @@ loop: buf.Write(data[:n]) if buf.Len() > 4 { - cmd := string(buf.Bytes()[:4]) + frags := strings.SplitN(strings.TrimRight(buf.String(), "\r\n"), " ", 2) + var cmd = frags[0] + var args string + if len(frags) > 1 { + args = frags[1] + } + log.Trace("FTP connection[%s] exec command: %s", conn.RemoteAddr(), strings.TrimRight(buf.String(), "\r\n")) + + if _rule == nil && cmd != "USER" && cmd != "PASS" { + _, _ = conn.Write([]byte("332 Need account for login.\r\n")) + break loop + } + switch cmd { case "USER": - user = strings.TrimRight(string(buf.Bytes()[5:]), "\r\n") + user = args _, _ = conn.Write([]byte("331 password please - version check\r\n")) case "PASS": - password = strings.TrimRight(string(buf.Bytes()[5:]), "\r\n") - _, _ = conn.Write([]byte("230 User logged in\r\n")) - - for _, _rule := range s.getRules() { - for _, s := range []string{user, password} { - flag, flagGroup, vars = _rule.Match(s) - if flag != "" { - vars["user"] = user - vars["password"] = password - break - } - } - if flag == "" { - continue - } - matchedRule = _rule + password = args + if _rule, flag, flagGroup, vars = s.authenticate(user, password); _rule == nil { + _, _ = conn.Write([]byte("331 please specify the password\r\n")) + break loop } - case "QUIT": - _, _ = conn.Write([]byte("221 Goodbye.\r\n")) - case "RETR": - path += "/" + strings.TrimRight(string(buf.Bytes()[5:]), "\r\n") - _, _ = conn.Write([]byte("451 Nope\r\n")) - _, _ = conn.Write([]byte("221 Goodbye.\r\n")) - status = FINISHED - break loop + log.Trace("FTP connection[%s] matched rule[rule_name: %s, flag: %s]", conn.RemoteAddr(), _rule.Name, flag) + _, _ = conn.Write([]byte("230 User logged in\r\n")) + + if pasvAddress = s.getPasvAddressFromCache(ip, _rule.PasvAddress); pasvAddress == "" { + pasvAddress = fmt.Sprintf("%s:%d", s.PasvIP, s.PasvPort) + } + + isRedirect = rule.CompileTpl(pasvAddress, vars) != fmt.Sprintf("%s:%d", s.PasvIP, s.PasvPort) + case "SIZE": + path += strings.TrimLeft(args, "/") + if _rule == nil || isRedirect || len(_rule.Data) == 0 { + _, _ = conn.Write([]byte(fmt.Sprintf("550 %s: No such file or directory.\r\n", args))) + break + } + _, _ = conn.Write([]byte(fmt.Sprintf("213 %d\r\n", len(_rule.Data)))) case "EPSV", "EPRT", "PORT": // refuse to use EPSV/EPRT/PORT in order to make the client to use PASV mode. _, _ = conn.Write([]byte(fmt.Sprintf("500 '%s': command not understood.\r\n", cmd))) case "PASV": - // return rule's pasv_address or default pasv address - ret := fmt.Sprintf("227 Entering Passive Mode (%s,%v,%d)\r\n", strings.ReplaceAll(s.PasvIP, ".", ","), float64(s.PasvPort/256), s.PasvPort%256) - - if matchedRule != nil && matchedRule.PasvAddress != "" { - pasvAddress := rule.CompileTpl(matchedRule.PasvAddress, vars) + //Just so that ide does not prompt that there may be a nil value + if _rule != nil { + // return rule's pasv_address or default pasv address + pasvAddress := rule.CompileTpl(pasvAddress, vars) pasvIP, pasvPort, err := net.SplitHostPort(pasvAddress) if err != nil { - log.Warn("FTP failed to split rule[id%d] pasv_address(%s) :%s", matchedRule.ID, pasvAddress, err) + log.Warn("FTP failed to split rule[id%d] pasv_address(%s) :%s", _rule.ID, pasvAddress, err) break } + port, err := strconv.Atoi(pasvPort) if err != nil { - log.Warn("FTP failed to convert rule[id%d] pasv_port(%s) :%s", matchedRule.ID, pasvPort, err) + log.Warn("FTP failed to convert rule[id%d] pasv_port(%s) :%s", _rule.ID, pasvPort, err) break } - ret = fmt.Sprintf("227 Entering Passive Mode (%s,%v,%d)\r\n", strings.ReplaceAll(pasvIP, ".", ","), float64(port/256), port%256) + ret := fmt.Sprintf("227 Entering Passive Mode (%s,%v,%d)\r\n", strings.ReplaceAll(pasvIP, ".", ","), float64(port/256), port%256) + _, _ = conn.Write([]byte(ret)) + if isRedirect { + log.Trace("FTP connection[%s] will be redirect[pasv_address: %s]", conn.RemoteAddr(), pasvAddress) + } } - _, _ = conn.Write([]byte(ret)) + case "RETR": + //Just so that ide does not prompt that there may be a nil value + if _rule != nil { + filename = args + method = DOWNLOAD + + //send data to client + _, _ = conn.Write([]byte(fmt.Sprintf("150 Opening BINARY mode data connection for '%s' (%d bytes).\r\n", filename, len(_rule.Data)))) + s.dataChannel <- map[string]interface{}{clientPasvConnAddress: []byte(rule.CompileTpl(_rule.Data, vars))} + _, _ = conn.Write([]byte("226 Transfer complete.\r\n")) + } + + case "STOR": + filename = args + method = UPLOAD + + _, _ = conn.Write([]byte(fmt.Sprintf("150 Opening BINARY mode data connection for '%s'.\r\n", filename))) + //only could read data send to local pasv server. + if !isRedirect { + dataChannel := make(chan []byte) + s.dataChannel <- map[string]interface{}{clientPasvConnAddress: dataChannel} + uploadData = <-dataChannel + log.Trace("FTP connection[%s] uploaded %d bytes", conn.RemoteAddr(), len(uploadData)) + } + _, _ = conn.Write([]byte("226 Transfer complete.\r\n")) + case "QUIT": + _, _ = conn.Write([]byte("221 Goodbye.\r\n")) + status = FINISHED + break loop + case "CWD": + _, _ = conn.Write([]byte("250 Directory successfully changed.\r\n")) + path += strings.TrimRight(args, "\r\n") + "/" + case "PWD": + _, _ = conn.Write([]byte(fmt.Sprintf("257 \"%s\" is the current directory\r\n", path))) default: - cmd = string(buf.Bytes()[:3]) - if cmd == "CWD" { - _, _ = conn.Write([]byte("250 Directory successfully changed.\r\n")) - path += "/" + strings.TrimRight(string(buf.Bytes()[4:]), "\r\n") - } else if cmd == "PWD" { - _, _ = conn.Write([]byte("257 \"/\" is the current directory\r\n")) - } else { - _, _ = conn.Write([]byte("230 more data please!\r\n")) - } + _, _ = conn.Write([]byte("230 more data please!\r\n")) } } buf = &bytes.Buffer{} } - if matchedRule != nil { - _rule := matchedRule + if _rule != nil { area := qqwry.Area(ip) - + var r *Record + var err error // create new record - r, err := NewRecord(_rule, flag, user, password, path, ip, area, status) + ftpFile := &file.FTPFile{} + if len(uploadData) != 0 { + ftpFile = &file.FTPFile{ + Name: filename, + Content: uploadData, + } + } + + r, err = NewRecord(_rule, flag, user, password, method, path, ip, area, ftpFile, status) if err != nil { log.Warn("FTP record[rule_id:%d] created failed :%s", _rule.ID, err) return @@ -193,19 +288,35 @@ loop: } } +func (s *Server) handlePasvConnection(conn net.Conn, data map[string]interface{}) { + remoteAddress := conn.RemoteAddr().String() + switch v := data[remoteAddress].(type) { + case types.Nil: + s.dataChannel <- data + return + case []byte: + _, err := conn.Write(v) + if err != nil { + log.Warn("FTP PASV server sent data to connection[%s] failed with error: %s", remoteAddress, err) + } + log.Trace("FTP PASV server has sent data to connection[%s]", remoteAddress) + case chan []byte: + buf, err := io.ReadAll(conn) + if err != nil { + log.Warn("FTP PASV server received data from connection[%s] failed with error: %s", remoteAddress, err) + } + v <- buf + log.Trace("FTP PASV server has received data from connection[%s]", remoteAddress) + } + + _ = conn.Close() +} + func (s *Server) Run() { if err := s.updateRules(); err != nil { log.Fatal(err.Error()) } - - // run server - log.Info("Starting FTP Server at %v", s.Addr) - - listener, err := net.Listen("tcp", s.Addr) - if err != nil { - log.Fatal(err.Error()) - } - + // run pasv server go func() { pasvAddress := fmt.Sprintf("%s:%d", strings.Split(s.Addr, ":")[0], s.PasvPort) log.Info("Start to listen FTP PASV port at %v", pasvAddress) @@ -213,16 +324,23 @@ func (s *Server) Run() { if err != nil { log.Fatal("FTP failed to listen on pasv port : %v", err) } - for { + for data := range s.dataChannel { tcpConn, err := listener.Accept() if err != nil { log.Warn("FTP accept connection error: %v", err) continue } - _ = tcpConn.Close() + s.handlePasvConnection(tcpConn, data) } }() + // run ftp server + log.Info("Starting FTP Server at %v", s.Addr) + + listener, err := net.Listen("tcp", s.Addr) + if err != nil { + log.Fatal(err.Error()) + } for { tcpConn, err := listener.Accept() if err != nil { diff --git a/pkg/ftp/record.go b/pkg/ftp/record.go index 6a3cbac..aee4d6e 100644 --- a/pkg/ftp/record.go +++ b/pkg/ftp/record.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/li4n0/revsuit/internal/database" + "github.com/li4n0/revsuit/internal/file" "github.com/li4n0/revsuit/internal/notice" "github.com/li4n0/revsuit/internal/record" ) @@ -13,12 +14,14 @@ import ( var _ record.Record = (*Record)(nil) type Record struct { - User string `form:"user" json:"user"` - Password string `form:"password" json:"password"` - Path string `form:"path" json:"path"` - Status Status `form:"status" json:"status"` record.BaseRecord - Rule Rule `gorm:"foreignKey:RuleName;references:Name;constraint:OnUpdate:CASCADE,OnDelete:SET NULL;" form:"-" json:"-" notice:"-"` + User string `form:"user" json:"user"` + Password string `form:"password" json:"password"` + Path string `form:"path" json:"path"` + Method Method `form:"method" json:"method"` + Status Status `form:"status" json:"status"` + File *file.FTPFile `form:"file" json:"file" notice:"-"` + Rule Rule `gorm:"foreignKey:RuleName;references:Name;constraint:OnUpdate:CASCADE,OnDelete:SET NULL;" form:"-" json:"-" notice:"-"` } func (Record) TableName() string { @@ -29,7 +32,7 @@ func (r Record) Notice() { notice.Notice(r) } -func NewRecord(rule *Rule, flag, user, password, path, ip, area string, status Status) (r *Record, err error) { +func NewRecord(rule *Rule, flag, user, password, method, path, ip, area string, file *file.FTPFile, status Status) (r *Record, err error) { r = &Record{ BaseRecord: record.BaseRecord{ Flag: flag, @@ -38,9 +41,11 @@ func NewRecord(rule *Rule, flag, user, password, path, ip, area string, status S RequestTime: time.Now(), }, Path: path, + Method: method, User: user, Password: password, Status: status, + File: file, Rule: *rule, } return r, database.DB.Create(r).Error @@ -76,6 +81,9 @@ func ListRecords(c *gin.Context) { if ftpRecord.Path != "" { db.Where("path like ?", "%"+ftpRecord.Path+"%") } + if ftpRecord.Method != "" { + db.Where("method = ?", ftpRecord.Method) + } if ftpRecord.Status != "" { db.Where("status = ?", ftpRecord.Status) } @@ -100,7 +108,7 @@ func ListRecords(c *gin.Context) { order = "desc" } - if err := db.Order("id" + " " + order).Count(&count).Offset((page - 1) * 10).Limit(10).Find(&res).Error; err != nil { + if err := db.Preload("File").Order("id " + order).Count(&count).Offset((page - 1) * 10).Limit(10).Find(&res).Error; err != nil { c.JSON(400, gin.H{ "status": "failed", "error": err, diff --git a/pkg/ftp/rule.go b/pkg/ftp/rule.go index 09df82c..9b4a8a6 100644 --- a/pkg/ftp/rule.go +++ b/pkg/ftp/rule.go @@ -14,6 +14,7 @@ import ( type Rule struct { rule.BaseRule PasvAddress string `gorm:"pasv_address" json:"pasv_address" form:"pasv_address"` + Data []byte `json:"data" form:"data"` } func (Rule) TableName() string { @@ -44,6 +45,7 @@ func (r *Rule) CreateOrUpdate() (err error) { "flag_format", "rank", "pasv_address", + "data", "push_to_client", "notice", }), diff --git a/pkg/mysql/mysql.go b/pkg/mysql/mysql.go index 1c18a5f..36d9fd6 100644 --- a/pkg/mysql/mysql.go +++ b/pkg/mysql/mysql.go @@ -7,8 +7,10 @@ import ( "regexp" "strings" "sync" + "time" "github.com/li4n0/revsuit/internal/database" + "github.com/li4n0/revsuit/internal/file" "github.com/li4n0/revsuit/internal/qqwry" "github.com/li4n0/revsuit/pkg/mysql/vmysql" log "unknwon.dev/clog/v2" @@ -54,21 +56,29 @@ func (s *Server) updateRules() error { // NewConnection is part of the mysql.Handler interface. func (s *Server) NewConnection(c *vmysql.Conn) { - log.Trace("New MySQL client from addr [%s] logged in with username [%s], ID [%d]", c.RemoteAddr(), c.User, c.ConnectionID) + log.Trace("New MySQL connection from addr [%s] logged [%s] in with username [%s], ID [%d]", c.RemoteAddr(), c.SchemaName, c.User, c.ConnectionID) + + if err := c.Conn.SetDeadline(time.Now().Add(time.Second * 30)); err != nil { + log.Warn("MySQL set connection deadline error:%v", err) + } c.RecycleReadPacket() var ( user = c.User schema = c.SchemaName validated bool + flag string ) for _, _rule := range s.getRules() { - userFlag, _, _ := _rule.Match(user) - schemaFlag, _, _ := _rule.Match(schema) - if userFlag == "" && schemaFlag == "" { + flag, _, _ = _rule.Match(user) + if flag == "" { + flag, _, _ = _rule.Match(schema) + } + if flag == "" { continue } + log.Trace("MySQL connection[id: %d] matched rule[rule_name: %s, flag: %s]", c.ConnectionID, _rule.Name, flag) s.connRulePool.Store(c.ConnectionID, _rule) validated = true break @@ -117,17 +127,17 @@ func (s *Server) ConnectionClosed(c *vmysql.Conn) { ip := strings.Split(c.RemoteAddr().String(), ":")[0] - filenames := strings.Split(_rule.Files, FILE_SPEARATOR) - files := make([]File, 0) + filenames := strings.Split(_rule.Files, ",") + files := make([]file.MySQLFile, 0) for _, filename := range filenames { if len(c.Files[filename]) != 0 { - files = append(files, File{Name: filename, Content: c.Files[filename]}) + files = append(files, file.MySQLFile{Name: filename, Content: c.Files[filename]}) } } r, err := newRecord(_rule, flag, user, clientName, clientOS, ip, qqwry.Area(ip), supportLoadLocalData, files) if err != nil { - log.Warn("MySQL record(rule_id:%s) created failed :%s", _rule.Name, err) + log.Warn("MySQL record[rule_id: %s] created failed: %s", _rule.Name, err) return } log.Info("MySQL record[id:%d rule:%s remote_ip:%s] has been created", r.ID, _rule.Name, ip) @@ -151,7 +161,7 @@ func (s *Server) ConnectionClosed(c *vmysql.Conn) { if _rule.Notice { go func() { r.Notice() - log.Trace("MySQL record[id%d] notice has been sent", r.ID) + log.Trace("MySQL record[id: %d] notice has been sent", r.ID) }() } @@ -211,7 +221,7 @@ func (s *Server) ComQuery(c *vmysql.Conn, query string, callback func(*sqltypes. } // mysql LOAD DATA LOCAL - if !c.SupportLoadDataLocal { // 客户端不支持读取本地文件且没有开启总是读取，直接返回错误 + if !c.SupportLoadDataLocal { log.Trace("MySQL Client not support LOAD DATA LOCAL, return error directly") c.WriteErrorResponse( fmt.Sprintf( @@ -224,7 +234,7 @@ func (s *Server) ComQuery(c *vmysql.Conn, query string, callback func(*sqltypes. return nil } - files := strings.Split(_rule.Files, ";") + files := strings.Split(_rule.Files, ",") if c.Files == nil { c.Files = make(map[string][]byte) } @@ -238,12 +248,6 @@ func (s *Server) ComQuery(c *vmysql.Conn, query string, callback func(*sqltypes. } else { c.Files[filename] = data } - c.WriteErrorResponse(fmt.Sprintf( - "You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '%s' at line 1", - strings.ReplaceAll( - strings.ReplaceAll(query, "%", "%%"), - "'", "\\'"), - )) } } diff --git a/pkg/mysql/record.go b/pkg/mysql/record.go index 12ca7dd..1c8a635 100644 --- a/pkg/mysql/record.go +++ b/pkg/mysql/record.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/li4n0/revsuit/internal/database" + "github.com/li4n0/revsuit/internal/file" "github.com/li4n0/revsuit/internal/notice" "github.com/li4n0/revsuit/internal/record" ) @@ -14,12 +15,12 @@ var _ record.Record = (*Record)(nil) type Record struct { record.BaseRecord - Username string `gorm:"index" form:"username" json:"username" notice:"username"` - ClientName string `gorm:"index" form:"client_name" json:"client_name" notice:"client_name"` - ClientOS string `gorm:"index" form:"client_os" json:"client_os" notice:"client_os"` - LoadLocalData bool `gorm:"index" form:"load_local_data" json:"load_local_data" notice:"load_local_data"` - Files []File `form:"-" json:"files" notice:"-"` - Rule Rule `gorm:"foreignKey:RuleName;references:Name;constraint:OnUpdate:CASCADE,OnDelete:SET NULL;" form:"-" json:"-" notice:"-"` + Username string `gorm:"index" form:"username" json:"username" notice:"username"` + ClientName string `gorm:"index" form:"client_name" json:"client_name" notice:"client_name"` + ClientOS string `gorm:"index" form:"client_os" json:"client_os" notice:"client_os"` + LoadLocalData bool `gorm:"index" form:"load_local_data" json:"load_local_data" notice:"load_local_data"` + Files []file.MySQLFile `form:"-" json:"files" notice:"-"` + Rule Rule `gorm:"foreignKey:RuleName;references:Name;constraint:OnUpdate:CASCADE,OnDelete:SET NULL;" form:"-" json:"-" notice:"-"` } func (Record) TableName() string { @@ -30,7 +31,7 @@ func (r Record) Notice() { notice.Notice(r) } -func newRecord(rule *Rule, flag, username, clientName, clientOS, remoteIp, ipArea string, supportLoadLocalData bool, files []File) (r *Record, err error) { +func newRecord(rule *Rule, flag, username, clientName, clientOS, remoteIp, ipArea string, supportLoadLocalData bool, files []file.MySQLFile) (r *Record, err error) { r = &Record{ BaseRecord: record.BaseRecord{ Flag: flag, diff --git a/pkg/mysql/vmysql/conn.go b/pkg/mysql/vmysql/conn.go index dc17242..76445b6 100644 --- a/pkg/mysql/vmysql/conn.go +++ b/pkg/mysql/vmysql/conn.go @@ -73,7 +73,7 @@ type Conn struct { // conn is the underlying network connection. // Calling Close() on the Conn will close this connection. // If there are any ongoing reads or writes, they may get interrupted. - conn net.Conn + Conn net.Conn // For server-side connections, listener points to the server object. listener *Listener @@ -167,7 +167,7 @@ var writersPool = sync.Pool{New: func() interface{} { return bufio.NewWriterSize // size for reads. func newServerConn(conn net.Conn, listener *Listener) *Conn { c := &Conn{ - conn: conn, + Conn: conn, listener: listener, closed: sync2.NewAtomicBool(false), } @@ -181,7 +181,7 @@ func newServerConn(conn net.Conn, listener *Listener) *Conn { // be terminated by a call to flush. func (c *Conn) startWriterBuffering() { c.bufferedWriter = writersPool.Get().(*bufio.Writer) - c.bufferedWriter.Reset(c.conn) + c.bufferedWriter.Reset(c.Conn) } // flush flushes the written data to the socket. @@ -206,7 +206,7 @@ func (c *Conn) getWriter() io.Writer { if c.bufferedWriter != nil { return c.bufferedWriter } - return c.conn + return c.Conn } // getReader returns reader for connection. It can be *bufio.Reader or net.Conn @@ -215,7 +215,7 @@ func (c *Conn) getReader() io.Reader { if c.bufferedReader != nil { return c.bufferedReader } - return c.conn + return c.Conn } func (c *Conn) readHeaderFrom(r io.Reader) (int, error) { @@ -359,10 +359,10 @@ func (c *Conn) readUploadFileEphemeralPacket() []byte { // This function usually shouldn't be used - use readEphemeralPacket. func (c *Conn) readEphemeralPacketDirect() ([]byte, error) { if c.currentEphemeralPolicy != ephemeralUnused { - panic(vterrors.Errorf(vtrpc.Code_INTERNAL, "readEphemeralPacketDirect: unexpected currentEphemeralPolicy: %v", c.currentEphemeralPolicy)) + return nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "readEphemeralPacketDirect: unexpected currentEphemeralPolicy: %v", c.currentEphemeralPolicy) } - var r io.Reader = c.conn + var r io.Reader = c.Conn length, err := c.readHeaderFrom(r) if err != nil { @@ -577,7 +577,7 @@ func (c *Conn) recycleWritePacket() { // RemoteAddr returns the underlying socket RemoteAddr(). func (c *Conn) RemoteAddr() net.Addr { - return c.conn.RemoteAddr() + return c.Conn.RemoteAddr() } // ID returns the MySQL connection ID for this connection. @@ -594,7 +594,7 @@ func (c *Conn) String() string { // routine to interrupt the current connection. func (c *Conn) Close() { if c.closed.CompareAndSwap(false, true) { - c.conn.Close() + c.Conn.Close() } } diff --git a/pkg/mysql/vmysql/server.go b/pkg/mysql/vmysql/server.go index 3c56f42..157f2e4 100644 --- a/pkg/mysql/vmysql/server.go +++ b/pkg/mysql/vmysql/server.go @@ -317,7 +317,7 @@ func (l *Listener) handle(conn net.Conn, connectionID uint32, acceptTime time.Ti } c.RecycleReadPacket() - if con, ok := c.conn.(*tls.Conn); ok { + if con, ok := c.Conn.(*tls.Conn); ok { connState := con.ConnectionState() tlsVerStr := tlsVersionToString(connState.Version) if tlsVerStr != "" { @@ -619,8 +619,8 @@ func (l *Listener) parseClientHandshakePacket(c *Conn, firstTime bool, data []by // Check for SSL. if firstTime && l.TLSConfig != nil && clientFlags&CapabilityClientSSL > 0 { // Need to switch to TLS, and then re-read the packet. - conn := tls.Server(c.conn, l.TLSConfig) - c.conn = conn + conn := tls.Server(c.Conn, l.TLSConfig) + c.Conn = conn c.bufferedReader.Reset(conn) c.Capabilities |= CapabilityClientSSL return "", "", nil, nil diff --git a/pkg/server/router.go b/pkg/server/router.go index 2148cf5..ffd8b43 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/li4n0/revsuit/frontend" + "github.com/li4n0/revsuit/internal/file" "github.com/li4n0/revsuit/pkg/dns" "github.com/li4n0/revsuit/pkg/ftp" "github.com/li4n0/revsuit/pkg/mysql" @@ -101,6 +102,6 @@ func (revsuit *Revsuit) registerHttpRouter() { // init file router group fileGroup := revsuit.http.ApiGroup.Group("/file") - fileGroup.GET("/mysql/:id", mysql.GetFile) + fileGroup.GET("/:record_type/:id", file.GetFile) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 1ac6c98..246a5df 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -3,6 +3,7 @@ package server import ( "github.com/gin-gonic/gin" "github.com/li4n0/revsuit/internal/database" + "github.com/li4n0/revsuit/internal/file" "github.com/li4n0/revsuit/internal/notice" "github.com/li4n0/revsuit/pkg/dns" "github.com/li4n0/revsuit/pkg/ftp" @@ -53,7 +54,7 @@ func initDatabase(dsn string) { if err != nil { log.Fatal(err.Error()) } - err = database.DB.AutoMigrate(&mysql.File{}) + err = database.DB.AutoMigrate(&file.MySQLFile{}) if err != nil { log.Fatal(err.Error()) } @@ -73,13 +74,17 @@ func initDatabase(dsn string) { if err != nil { log.Fatal(err.Error()) } + err = database.DB.AutoMigrate(&file.FTPFile{}) + if err != nil { + log.Fatal(err.Error()) + } } func initLog(level string) (logLevel log.Level) { switch level { - case "debug": + case "debug", "trace": gin.SetMode(gin.DebugMode) database.DB.Logger.LogMode(logger.Info) logLevel = log.LevelTrace @@ -87,7 +92,7 @@ func initLog(level string) (logLevel log.Level) { gin.SetMode(gin.DebugMode) database.DB.Logger.LogMode(logger.Info) logLevel = log.LevelInfo - case "warning": + case "warning", "warn": gin.SetMode(gin.ReleaseMode) database.DB.Logger.LogMode(logger.Warn) logLevel = log.LevelWarn @@ -133,8 +138,8 @@ func initNotice(nc noticeConfig) { func New(c *Config) *Revsuit { - initDatabase(c.Database) logLevel := initLog(c.LogLevel) + initDatabase(c.Database) initNotice(c.Notice) s := &Revsuit{