From 7d7818bc3892950a0791b8aee18239d1d74de203 Mon Sep 17 00:00:00 2001 From: Gordon <1432970085@qq.com> Date: Mon, 11 Oct 2021 18:18:50 +0800 Subject: [PATCH] log and scripts optimization --- config/config.yaml | 7 +- script/msg_gateway_start.sh | 2 +- script/msg_transfer_start.sh | 2 +- script/push_start.sh | 2 +- script/sdk_svr_start.sh | 2 +- script/start_all.sh | 2 +- src/api/auth/user_register.go | 2 +- src/api/auth/user_token.go | 2 +- src/api/open_im_api.go | 3 +- src/common/config/config.go | 3 +- src/common/log/logrus.go | 116 ++++++++++++++-------- src/rpc/auth/auth/rpcAuth.go | 18 ++-- src/rpc/chat/chat/rpcChat.go | 15 +-- src/rpc/friend/friend/get_firends_info.go | 1 + src/rpc/group/group/create_group.go | 1 + src/rpc/user/user/get_user_info.go | 1 + 16 files changed, 113 insertions(+), 66 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 03428a8ec..9e8741b48 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -88,12 +88,13 @@ rpcregistername: openImPushName: Push openImOnlineMessageRelayName: OnlineMessageRelay openImGroupName: Group - rpcGetTokenName: Auth + openImAuthName: Auth log: storageLocation: ../logs/ - rotationTime: 12 - remainRotationCount: 10 + rotationTime: 24 + remainRotationCount: 5 + remainLogLevel: 6 elasticSearchSwitch: false elasticSearchAddr: [ 127.0.0.1:9201 ] elasticSearchUser: "" diff --git a/script/msg_gateway_start.sh b/script/msg_gateway_start.sh index eca66e9bf..197c28743 100644 --- a/script/msg_gateway_start.sh +++ b/script/msg_gateway_start.sh @@ -26,7 +26,7 @@ fi sleep 1 cd ${msg_gateway_binary_root} for ((i = 0; i < ${#ws_ports[@]}; i++)); do - nohup ./${msg_gateway_name} -rpc_port ${rpc_ports[$i]} -ws_port ${ws_ports[$i]} >>../logs/${msg_gateway_name}.log 2>&1 & + nohup ./${msg_gateway_name} -rpc_port ${rpc_ports[$i]} -ws_port ${ws_ports[$i]} >>../logs/openIM.log 2>&1 & done #Check launched service process diff --git a/script/msg_transfer_start.sh b/script/msg_transfer_start.sh index 13af4d9bd..30caeb466 100644 --- a/script/msg_transfer_start.sh +++ b/script/msg_transfer_start.sh @@ -16,7 +16,7 @@ fi #Waiting port recycling sleep 1 cd ${msg_transfer_binary_root} - nohup ./${msg_transfer_name} >>../logs/${msg_transfer_name}.log 2>&1 & + nohup ./${msg_transfer_name} >>../logs/openIM.log 2>&1 & #Check launched service process check=`ps aux | grep -w ./${msg_transfer_name} | grep -v grep| wc -l` if [ $check -eq 1 ] diff --git a/script/push_start.sh b/script/push_start.sh index f636e99e1..668c0068f 100644 --- a/script/push_start.sh +++ b/script/push_start.sh @@ -22,7 +22,7 @@ sleep 1 cd ${push_binary_root} for ((i = 0; i < ${#rpc_ports[@]}; i++)); do - nohup ./${push_name} -port ${rpc_ports[$i]} >>../logs/${push_name}.log 2>&1 & + nohup ./${push_name} -port ${rpc_ports[$i]} >>../logs/openIM.log 2>&1 & done sleep 3 diff --git a/script/sdk_svr_start.sh b/script/sdk_svr_start.sh index 16c468fac..28d764b73 100644 --- a/script/sdk_svr_start.sh +++ b/script/sdk_svr_start.sh @@ -26,7 +26,7 @@ fi #Waiting port recycling sleep 1 cd ${sdk_server_binary_root} - nohup ./${sdk_server_name} -openIM_api_port ${api_ports[0]} -openIM_ws_port ${ws_ports[0]} -sdk_ws_port ${sdk_ws_ports[0]} >>../logs/${sdk_server_name}.log 2>&1 & + nohup ./${sdk_server_name} -openIM_api_port ${api_ports[0]} -openIM_ws_port ${ws_ports[0]} -sdk_ws_port ${sdk_ws_ports[0]} >>../logs/openIM.log 2>&1 & #Check launched service process sleep 3 diff --git a/script/start_all.sh b/script/start_all.sh index 37f76283f..43a02b34d 100644 --- a/script/start_all.sh +++ b/script/start_all.sh @@ -8,7 +8,7 @@ need_to_start_server_shell=( msg_gateway_start.sh push_start.sh msg_transfer_start.sh - start_sdk_svr.sh + sdk_svr_start.sh ) for i in ${need_to_start_server_shell[*]}; do diff --git a/src/api/auth/user_register.go b/src/api/auth/user_register.go index 72c567df9..8dc64bb74 100644 --- a/src/api/auth/user_register.go +++ b/src/api/auth/user_register.go @@ -40,7 +40,7 @@ func newUserRegisterReq(params *paramsUserRegister) *pbAuth.UserRegisterReq { func UserRegister(c *gin.Context) { log.Info("", "", "api user_register init ....") - etcdConn := getcdv3.GetConn(config.Config.Etcd.EtcdSchema, strings.Join(config.Config.Etcd.EtcdAddr, ","), config.Config.RpcRegisterName.RpcGetTokenName) + etcdConn := getcdv3.GetConn(config.Config.Etcd.EtcdSchema, strings.Join(config.Config.Etcd.EtcdAddr, ","), config.Config.RpcRegisterName.OpenImAuthName) client := pbAuth.NewAuthClient(etcdConn) //defer etcdConn.Close() diff --git a/src/api/auth/user_token.go b/src/api/auth/user_token.go index 7f48e7f0b..a50369361 100644 --- a/src/api/auth/user_token.go +++ b/src/api/auth/user_token.go @@ -27,7 +27,7 @@ func newUserTokenReq(params *paramsUserToken) *pbAuth.UserTokenReq { func UserToken(c *gin.Context) { log.Info("", "", "api user_token init ....") - etcdConn := getcdv3.GetConn(config.Config.Etcd.EtcdSchema, strings.Join(config.Config.Etcd.EtcdAddr, ","), config.Config.RpcRegisterName.RpcGetTokenName) + etcdConn := getcdv3.GetConn(config.Config.Etcd.EtcdSchema, strings.Join(config.Config.Etcd.EtcdAddr, ","), config.Config.RpcRegisterName.OpenImAuthName) client := pbAuth.NewAuthClient(etcdConn) //defer etcdConn.Close() diff --git a/src/api/open_im_api.go b/src/api/open_im_api.go index eac0a1fa5..6065ea7e7 100644 --- a/src/api/open_im_api.go +++ b/src/api/open_im_api.go @@ -8,6 +8,7 @@ import ( "Open_IM/src/api/manage" apiThird "Open_IM/src/api/third" "Open_IM/src/api/user" + "Open_IM/src/common/log" "Open_IM/src/utils" "flag" "github.com/gin-gonic/gin" @@ -93,7 +94,7 @@ func main() { managementGroup.POST("/send_msg", manage.ManagementSendMsg) managementGroup.POST("/get_all_users_uid", manage.GetAllUsersUid) } - + log.NewPrivateLog("api") ginPort := flag.Int("port", 10000, "get ginServerPort from cmd,default 10000 as port") flag.Parse() r.Run(utils.ServerIP + ":" + strconv.Itoa(*ginPort)) diff --git a/src/common/config/config.go b/src/common/config/config.go index b11eb638c..faf75a5b8 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -71,7 +71,7 @@ type config struct { OpenImPushName string `yaml:"openImPushName"` OpenImOnlineMessageRelayName string `yaml:"openImOnlineMessageRelayName"` OpenImGroupName string `yaml:"openImGroupName"` - RpcGetTokenName string `yaml:"rpcGetTokenName"` + OpenImAuthName string `yaml:"openImAuthName"` } Etcd struct { EtcdSchema string `yaml:"etcdSchema"` @@ -81,6 +81,7 @@ type config struct { StorageLocation string `yaml:"storageLocation"` RotationTime int `yaml:"rotationTime"` RemainRotationCount uint `yaml:"remainRotationCount"` + RemainLogLevel uint `yaml:"remainLogLevel"` ElasticSearchSwitch bool `yaml:"elasticSearchSwitch"` ElasticSearchAddr []string `yaml:"elasticSearchAddr"` ElasticSearchUser string `yaml:"elasticSearchUser"` diff --git a/src/common/log/logrus.go b/src/common/log/logrus.go index 48c781c68..22720c618 100644 --- a/src/common/log/logrus.go +++ b/src/common/log/logrus.go @@ -2,6 +2,7 @@ package log import ( "Open_IM/src/common/config" + "bufio" "fmt" nested "github.com/antonfisher/nested-logrus-formatter" rotatelogs "github.com/lestrrat-go/file-rotatelogs" @@ -29,18 +30,25 @@ func NewPrivateLog(moduleName string) { func loggerInit(moduleName string) *Logger { var logger = logrus.New() //All logs will be printed - logger.SetLevel(logrus.TraceLevel) - //Log Style Setting + logger.SetLevel(logrus.Level(config.Config.Log.RemainLogLevel)) + //Close std console output + src, err := os.OpenFile(os.DevNull, os.O_APPEND|os.O_WRONLY, os.ModeAppend) + if err != nil { + panic(err) + } + writer := bufio.NewWriter(src) + logger.SetOutput(writer) + //Log Console Print Style Setting logger.SetFormatter(&nested.Formatter{ TimestampFormat: "2006-01-02 15:04:05.000", HideKeys: false, - FieldsOrder: []string{"PID"}, + FieldsOrder: []string{"PID", "FilePath", "OperationID"}, }) //File name and line number display hook logger.AddHook(newFileHook()) //Send logs to elasticsearch hook - if config.Config.Log.ElasticSearchSwitch == true { + if config.Config.Log.ElasticSearchSwitch { logger.AddHook(newEsHook(moduleName)) } //Log file segmentation hook @@ -60,13 +68,16 @@ func NewLfsHook(rotationTime time.Duration, maxRemainNum uint, moduleName string }, &nested.Formatter{ TimestampFormat: "2006-01-02 15:04:05.000", HideKeys: false, - FieldsOrder: []string{"PID"}, + FieldsOrder: []string{"PID", "FilePath", "OperationID"}, }) return lfsHook } func initRotateLogs(rotationTime time.Duration, maxRemainNum uint, level string, moduleName string) *rotatelogs.RotateLogs { + if moduleName != "" { + moduleName = moduleName + "." + } writer, err := rotatelogs.New( - config.Config.Log.StorageLocation+moduleName+"/"+level+"."+"%Y-%m-%d_%H-%M-%S", + config.Config.Log.StorageLocation+moduleName+level+"."+"%Y-%m-%d", rotatelogs.WithRotationTime(rotationTime), rotatelogs.WithRotationCount(maxRemainNum), ) @@ -77,54 +88,50 @@ func initRotateLogs(rotationTime time.Duration, maxRemainNum uint, level string, } } +//Deprecated func Info(token, OperationID, format string, args ...interface{}) { - if token == "" && OperationID == "" { - logger.WithFields(logrus.Fields{}).Infof(format, args...) - } else { - logger.WithFields(logrus.Fields{ - "token": token, - "OperationID": OperationID, - }).Infof(format, args...) - } + logger.WithFields(logrus.Fields{ + "PID": logger.Pid, + "OperationID": OperationID, + }).Infof(format, args...) + } +//Deprecated func Error(token, OperationID, format string, args ...interface{}) { - if token == "" && OperationID == "" { - logger.WithFields(logrus.Fields{}).Errorf(format, args...) - } else { - logger.WithFields(logrus.Fields{ - "token": token, - "OperationID": OperationID, - }).Errorf(format, args...) - } + + logger.WithFields(logrus.Fields{ + "PID": logger.Pid, + "OperationID": OperationID, + }).Errorf(format, args...) + } +//Deprecated func Debug(token, OperationID, format string, args ...interface{}) { - if token == "" && OperationID == "" { - logger.WithFields(logrus.Fields{}).Debugf(format, args...) - } else { - logger.WithFields(logrus.Fields{ - "token": token, - "OperationID": OperationID, - }).Debugf(format, args...) - } + + logger.WithFields(logrus.Fields{ + "PID": logger.Pid, + "OperationID": OperationID, + }).Debugf(format, args...) + } +//Deprecated func Warning(token, OperationID, format string, args ...interface{}) { - if token == "" && OperationID == "" { - logger.WithFields(logrus.Fields{}).Warningf(format, args...) - } else { - logger.WithFields(logrus.Fields{ - "token": token, - "OperationID": OperationID, - }).Warningf(format, args...) - } + logger.WithFields(logrus.Fields{ + "PID": logger.Pid, + "OperationID": OperationID, + }).Warningf(format, args...) + } +//Deprecated func InfoByArgs(format string, args ...interface{}) { logger.WithFields(logrus.Fields{}).Infof(format, args) } +//Deprecated func ErrorByArgs(format string, args ...interface{}) { logger.WithFields(logrus.Fields{}).Errorf(format, args...) } @@ -132,21 +139,28 @@ func ErrorByArgs(format string, args ...interface{}) { //Print log information in k, v format, //kv is best to appear in pairs. tipInfo is the log prompt information for printing, //and kv is the key and value for printing. +//Deprecated func InfoByKv(tipInfo, OperationID string, args ...interface{}) { fields := make(logrus.Fields) argsHandle(OperationID, fields, args) logger.WithFields(fields).Info(tipInfo) } + +//Deprecated func ErrorByKv(tipInfo, OperationID string, args ...interface{}) { fields := make(logrus.Fields) argsHandle(OperationID, fields, args) logger.WithFields(fields).Error(tipInfo) } + +//Deprecated func DebugByKv(tipInfo, OperationID string, args ...interface{}) { fields := make(logrus.Fields) argsHandle(OperationID, fields, args) logger.WithFields(fields).Debug(tipInfo) } + +//Deprecated func WarnByKv(tipInfo, OperationID string, args ...interface{}) { fields := make(logrus.Fields) argsHandle(OperationID, fields, args) @@ -162,6 +176,30 @@ func argsHandle(OperationID string, fields logrus.Fields, args []interface{}) { fields[fmt.Sprintf("%v", args[i])] = "" } } - fields["operationID"] = OperationID + fields["OperationID"] = OperationID fields["PID"] = logger.Pid } +func NewInfo(OperationID string, args ...interface{}) { + logger.WithFields(logrus.Fields{ + "OperationID": OperationID, + "PID": logger.Pid, + }).Infoln(args) +} +func NewError(OperationID string, args ...interface{}) { + logger.WithFields(logrus.Fields{ + "OperationID": OperationID, + "PID": logger.Pid, + }).Errorln(args) +} +func NewDebug(OperationID string, args ...interface{}) { + logger.WithFields(logrus.Fields{ + "OperationID": OperationID, + "PID": logger.Pid, + }).Debugln(args) +} +func NewWarn(OperationID string, args ...interface{}) { + logger.WithFields(logrus.Fields{ + "OperationID": OperationID, + "PID": logger.Pid, + }).Warnln(args) +} diff --git a/src/rpc/auth/auth/rpcAuth.go b/src/rpc/auth/auth/rpcAuth.go index 237fc9bab..8c83a97fd 100644 --- a/src/rpc/auth/auth/rpcAuth.go +++ b/src/rpc/auth/auth/rpcAuth.go @@ -2,7 +2,8 @@ package rpcAuth import ( "Open_IM/src/common/config" - log2 "Open_IM/src/common/log" + "Open_IM/src/common/log" + "Open_IM/src/grpc-etcdv3/getcdv3" pbAuth "Open_IM/src/proto/auth" "Open_IM/src/utils" @@ -20,24 +21,25 @@ type rpcAuth struct { } func NewRpcAuthServer(port int) *rpcAuth { + log.NewPrivateLog("auth") return &rpcAuth{ rpcPort: port, - rpcRegisterName: config.Config.RpcRegisterName.RpcGetTokenName, + rpcRegisterName: config.Config.RpcRegisterName.OpenImAuthName, etcdSchema: config.Config.Etcd.EtcdSchema, etcdAddr: config.Config.Etcd.EtcdAddr, } } func (rpc *rpcAuth) Run() { - log2.Info("", "", "rpc get_token init...") + log.Info("", "", "rpc get_token init...") address := utils.ServerIP + ":" + strconv.Itoa(rpc.rpcPort) listener, err := net.Listen("tcp", address) if err != nil { - log2.Error("", "", "listen network failed, err = %s, address = %s", err.Error(), address) + log.Error("", "", "listen network failed, err = %s, address = %s", err.Error(), address) return } - log2.Info("", "", "listen network success, address = %s", address) + log.Info("", "", "listen network success, address = %s", address) //grpc server srv := grpc.NewServer() @@ -48,14 +50,14 @@ func (rpc *rpcAuth) Run() { pbAuth.RegisterAuthServer(srv, rpc) err = getcdv3.RegisterEtcd(rpc.etcdSchema, strings.Join(rpc.etcdAddr, ","), utils.ServerIP, rpc.rpcPort, rpc.rpcRegisterName, 10) if err != nil { - log2.Error("", "", "register rpc get_token to etcd failed, err = %s", err.Error()) + log.Error("", "", "register rpc get_token to etcd failed, err = %s", err.Error()) return } err = srv.Serve(listener) if err != nil { - log2.Info("", "", "rpc get_token fail, err = %s", err.Error()) + log.Info("", "", "rpc get_token fail, err = %s", err.Error()) return } - log2.Info("", "", "rpc get_token init success") + log.Info("", "", "rpc get_token init success") } diff --git a/src/rpc/chat/chat/rpcChat.go b/src/rpc/chat/chat/rpcChat.go index ca5aeefa0..d1dbad54b 100644 --- a/src/rpc/chat/chat/rpcChat.go +++ b/src/rpc/chat/chat/rpcChat.go @@ -3,7 +3,7 @@ package rpcChat import ( "Open_IM/src/common/config" "Open_IM/src/common/kafka" - log2 "Open_IM/src/common/log" + "Open_IM/src/common/log" "Open_IM/src/grpc-etcdv3/getcdv3" pbChat "Open_IM/src/proto/chat" "Open_IM/src/utils" @@ -22,6 +22,7 @@ type rpcChat struct { } func NewRpcChatServer(port int) *rpcChat { + log.NewPrivateLog("msg") rc := rpcChat{ rpcPort: port, rpcRegisterName: config.Config.RpcRegisterName.OpenImOfflineMessageName, @@ -33,15 +34,15 @@ func NewRpcChatServer(port int) *rpcChat { } func (rpc *rpcChat) Run() { - log2.Info("", "", "rpc get_token init...") + log.Info("", "", "rpc get_token init...") address := utils.ServerIP + ":" + strconv.Itoa(rpc.rpcPort) listener, err := net.Listen("tcp", address) if err != nil { - log2.Error("", "", "listen network failed, err = %s, address = %s", err.Error(), address) + log.Error("", "", "listen network failed, err = %s, address = %s", err.Error(), address) return } - log2.Info("", "", "listen network success, address = %s", address) + log.Info("", "", "listen network success, address = %s", address) //grpc server srv := grpc.NewServer() @@ -52,14 +53,14 @@ func (rpc *rpcChat) Run() { pbChat.RegisterChatServer(srv, rpc) err = getcdv3.RegisterEtcd(rpc.etcdSchema, strings.Join(rpc.etcdAddr, ","), utils.ServerIP, rpc.rpcPort, rpc.rpcRegisterName, 10) if err != nil { - log2.Error("", "", "register rpc get_token to etcd failed, err = %s", err.Error()) + log.Error("", "", "register rpc get_token to etcd failed, err = %s", err.Error()) return } err = srv.Serve(listener) if err != nil { - log2.Info("", "", "rpc get_token fail, err = %s", err.Error()) + log.Info("", "", "rpc get_token fail, err = %s", err.Error()) return } - log2.Info("", "", "rpc get_token init success") + log.Info("", "", "rpc get_token init success") } diff --git a/src/rpc/friend/friend/get_firends_info.go b/src/rpc/friend/friend/get_firends_info.go index 1da6ebaaf..a090a2b22 100644 --- a/src/rpc/friend/friend/get_firends_info.go +++ b/src/rpc/friend/friend/get_firends_info.go @@ -24,6 +24,7 @@ type friendServer struct { } func NewFriendServer(port int) *friendServer { + log.NewPrivateLog("friend") return &friendServer{ rpcPort: port, rpcRegisterName: config.Config.RpcRegisterName.OpenImFriendName, diff --git a/src/rpc/group/group/create_group.go b/src/rpc/group/group/create_group.go index 4aeb073d7..2d558f5d0 100644 --- a/src/rpc/group/group/create_group.go +++ b/src/rpc/group/group/create_group.go @@ -28,6 +28,7 @@ type groupServer struct { } func NewGroupServer(port int) *groupServer { + log.NewPrivateLog("group") return &groupServer{ rpcPort: port, rpcRegisterName: config.Config.RpcRegisterName.OpenImGroupName, diff --git a/src/rpc/user/user/get_user_info.go b/src/rpc/user/user/get_user_info.go index 74ee468c4..f8d3e2065 100644 --- a/src/rpc/user/user/get_user_info.go +++ b/src/rpc/user/user/get_user_info.go @@ -22,6 +22,7 @@ type userServer struct { } func NewUserServer(port int) *userServer { + log.NewPrivateLog("user") return &userServer{ rpcPort: port, rpcRegisterName: config.Config.RpcRegisterName.OpenImUserName,