diff --git a/log/format.go b/log/format.go index be350c5a6d..0a633e5cc7 100644 --- a/log/format.go +++ b/log/format.go @@ -77,11 +77,11 @@ type TerminalStringer interface { // a terminal with color-coded level output and terser human friendly timestamp. // This format should only be used for interactive programs or while developing. // -// [TIME] [LEVEL] MESAGE key=value key=value ... +// [LEVEL] [TIME] MESAGE key=value key=value ... // // Example: // -// [May 16 20:58:45] [DBUG] remove route ns=haproxy addr=127.0.0.1:50002 +// [DBUG] [May 16 20:58:45] remove route ns=haproxy addr=127.0.0.1:50002 // func TerminalFormat(usecolor bool) Format { return FormatFunc(func(r *Record) []byte { @@ -202,6 +202,48 @@ func JSONFormat() Format { return JSONFormatEx(false, true) } +// JSONFormatOrderedEx formats log records as JSON arrays. If pretty is true, +// records will be pretty-printed. If lineSeparated is true, records +// will be logged with a new line between each record. +func JSONFormatOrderedEx(pretty, lineSeparated bool) Format { + jsonMarshal := json.Marshal + if pretty { + jsonMarshal = func(v interface{}) ([]byte, error) { + return json.MarshalIndent(v, "", " ") + } + } + return FormatFunc(func(r *Record) []byte { + props := make(map[string]interface{}) + + props[r.KeyNames.Time] = r.Time + props[r.KeyNames.Lvl] = r.Lvl.String() + props[r.KeyNames.Msg] = r.Msg + + ctx := make([]string, len(r.Ctx)) + for i := 0; i < len(r.Ctx); i += 2 { + k, ok := r.Ctx[i].(string) + if !ok { + props[errorKey] = fmt.Sprintf("%+v is not a string key,", r.Ctx[i]) + } + ctx[i] = k + ctx[i+1] = formatLogfmtValue(r.Ctx[i+1], true) + } + props[r.KeyNames.Ctx] = ctx + + b, err := jsonMarshal(props) + if err != nil { + b, _ = jsonMarshal(map[string]string{ + errorKey: err.Error(), + }) + return b + } + if lineSeparated { + b = append(b, '\n') + } + return b + }) +} + // JSONFormatEx formats log records as JSON objects. If pretty is true, // records will be pretty-printed. If lineSeparated is true, records // will be logged with a new line between each record. diff --git a/log/handler.go b/log/handler.go index 3c99114dcb..6a0a25c15e 100644 --- a/log/handler.go +++ b/log/handler.go @@ -3,9 +3,13 @@ package log import ( "fmt" "io" + "io/ioutil" "net" "os" + "path/filepath" "reflect" + "regexp" + "strings" "sync" "github.com/go-stack/stack" @@ -70,6 +74,111 @@ func FileHandler(path string, fmtr Format) (Handler, error) { return closingHandler{f, StreamHandler(f, fmtr)}, nil } +// countingWriter wraps a WriteCloser object in order to count the written bytes. +type countingWriter struct { + w io.WriteCloser // the wrapped object + count uint // number of bytes written +} + +// Write increments the byte counter by the number of bytes written. +// Implements the WriteCloser interface. +func (w *countingWriter) Write(p []byte) (n int, err error) { + n, err = w.w.Write(p) + w.count += uint(n) + return n, err +} + +// Close implements the WriteCloser interface. +func (w *countingWriter) Close() error { + return w.w.Close() +} + +// prepFile opens the log file at the given path, and cuts off the invalid part +// from the end, because the previous execution could have been finished by interruption. +// Assumes that every line ended by '\n' contains a valid log record. +func prepFile(path string) (*countingWriter, error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0600) + if err != nil { + return nil, err + } + _, err = f.Seek(-1, io.SeekEnd) + if err != nil { + return nil, err + } + buf := make([]byte, 1) + var cut int64 + for { + if _, err := f.Read(buf); err != nil { + return nil, err + } + if buf[0] == '\n' { + break + } + if _, err = f.Seek(-2, io.SeekCurrent); err != nil { + return nil, err + } + cut++ + } + fi, err := f.Stat() + if err != nil { + return nil, err + } + ns := fi.Size() - cut + if err = f.Truncate(ns); err != nil { + return nil, err + } + return &countingWriter{w: f, count: uint(ns)}, nil +} + +// RotatingFileHandler returns a handler which writes log records to file chunks +// at the given path. When a file's size reaches the limit, the handler creates +// a new file named after the timestamp of the first log record it will contain. +func RotatingFileHandler(path string, limit uint, formatter Format) (Handler, error) { + if err := os.MkdirAll(path, 0700); err != nil { + return nil, err + } + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, err + } + re := regexp.MustCompile(`\.log$`) + last := len(files) - 1 + for last >= 0 && (!files[last].Mode().IsRegular() || !re.MatchString(files[last].Name())) { + last-- + } + var counter *countingWriter + if last >= 0 && files[last].Size() < int64(limit) { + // Open the last file, and continue to write into it until it's size reaches the limit. + if counter, err = prepFile(filepath.Join(path, files[last].Name())); err != nil { + return nil, err + } + } + if counter == nil { + counter = new(countingWriter) + } + h := StreamHandler(counter, formatter) + + return FuncHandler(func(r *Record) error { + if counter.count > limit { + counter.Close() + counter.w = nil + } + if counter.w == nil { + f, err := os.OpenFile( + filepath.Join(path, fmt.Sprintf("%s.log", strings.Replace(r.Time.Format("060102150405.00"), ".", "", 1))), + os.O_CREATE|os.O_APPEND|os.O_WRONLY, + 0600, + ) + if err != nil { + return err + } + counter.w = f + counter.count = 0 + } + return h.Log(r) + }), nil +} + // NetHandler opens a socket to the given address and writes records // over the connection. func NetHandler(network, addr string, fmtr Format) (Handler, error) { @@ -135,15 +244,14 @@ func CallerStackHandler(format string, h Handler) Handler { // wrapped Handler if the given function evaluates true. For example, // to only log records where the 'err' key is not nil: // -// logger.SetHandler(FilterHandler(func(r *Record) bool { -// for i := 0; i < len(r.Ctx); i += 2 { -// if r.Ctx[i] == "err" { -// return r.Ctx[i+1] != nil -// } -// } -// return false -// }, h)) -// +// logger.SetHandler(FilterHandler(func(r *Record) bool { +// for i := 0; i < len(r.Ctx); i += 2 { +// if r.Ctx[i] == "err" { +// return r.Ctx[i+1] != nil +// } +// } +// return false +// }, h)) func FilterHandler(fn func(r *Record) bool, h Handler) Handler { return FuncHandler(func(r *Record) error { if fn(r) { @@ -158,8 +266,7 @@ func FilterHandler(fn func(r *Record) bool, h Handler) Handler { // context matches the value. For example, to only log records // from your ui package: // -// log.MatchFilterHandler("pkg", "app/ui", log.StdoutHandler) -// +// log.MatchFilterHandler("pkg", "app/ui", log.StdoutHandler) func MatchFilterHandler(key string, value interface{}, h Handler) Handler { return FilterHandler(func(r *Record) (pass bool) { switch key { @@ -185,8 +292,7 @@ func MatchFilterHandler(key string, value interface{}, h Handler) Handler { // level to the wrapped Handler. For example, to only // log Error/Crit records: // -// log.LvlFilterHandler(log.LvlError, log.StdoutHandler) -// +// log.LvlFilterHandler(log.LvlError, log.StdoutHandler) func LvlFilterHandler(maxLvl Lvl, h Handler) Handler { return FilterHandler(func(r *Record) (pass bool) { return r.Lvl <= maxLvl @@ -198,10 +304,9 @@ func LvlFilterHandler(maxLvl Lvl, h Handler) Handler { // to different locations. For example, to log to a file and // standard error: // -// log.MultiHandler( -// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), -// log.StderrHandler) -// +// log.MultiHandler( +// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), +// log.StderrHandler) func MultiHandler(hs ...Handler) Handler { return FuncHandler(func(r *Record) error { for _, h := range hs { @@ -219,10 +324,10 @@ func MultiHandler(hs ...Handler) Handler { // to writing to a file if the network fails, and then to // standard out if the file write fails: // -// log.FailoverHandler( -// log.Must.NetHandler("tcp", ":9090", log.JSONFormat()), -// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), -// log.StdoutHandler) +// log.FailoverHandler( +// log.Must.NetHandler("tcp", ":9090", log.JSONFormat()), +// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), +// log.StdoutHandler) // // All writes that do not go to the first handler will add context with keys of // the form "failover_err_{idx}" which explain the error encountered while diff --git a/log/handler_glog.go b/log/handler_glog.go index f8b932fd1b..b372323082 100644 --- a/log/handler_glog.go +++ b/log/handler_glog.go @@ -57,6 +57,11 @@ func NewGlogHandler(h Handler) *GlogHandler { } } +// SetHandler updates the handler to write records to the specified sub-handler. +func (h *GlogHandler) SetHandler(nh Handler) { + h.origin = nh +} + // pattern contains a filter for the Vmodule option, holding a verbosity level // and a file pattern to match. type pattern struct { @@ -77,14 +82,14 @@ func (h *GlogHandler) Verbosity(level Lvl) { // // For instance: // -// pattern="gopher.go=3" -// sets the V level to 3 in all Go files named "gopher.go" +// pattern="gopher.go=3" +// sets the V level to 3 in all Go files named "gopher.go" // -// pattern="foo=3" -// sets V to 3 in all files of any packages whose import path ends in "foo" +// pattern="foo=3" +// sets V to 3 in all files of any packages whose import path ends in "foo" // -// pattern="foo/*=3" -// sets V to 3 in all files of any packages whose import path contains "foo" +// pattern="foo/*=3" +// sets V to 3 in all files of any packages whose import path contains "foo" func (h *GlogHandler) Vmodule(ruleset string) error { var filter []pattern for _, rule := range strings.Split(ruleset, ",") { diff --git a/log/logger.go b/log/logger.go index f678a13dcf..1d5e845db4 100644 --- a/log/logger.go +++ b/log/logger.go @@ -11,6 +11,7 @@ import ( const timeKey = "t" const lvlKey = "lvl" const msgKey = "msg" +const ctxKey = "ctx" const errorKey = "LOG15_ERROR" type Lvl int @@ -100,6 +101,7 @@ type RecordKeyNames struct { Time string Msg string Lvl string + Ctx string } // A Logger writes key/value pairs to a Handler @@ -138,6 +140,7 @@ func (l *logger) write(msg string, lvl Lvl, ctx []interface{}) { Time: timeKey, Msg: msgKey, Lvl: lvlKey, + Ctx: ctxKey, }, }) }