From be1764ef97d3e8022eca47679d323c44d83bbd93 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:30:12 +0800 Subject: [PATCH 001/143] etcd naming and discovery --- go.mod | 5 + go.sum | 11 ++ .../discoveryregister/discoveryregister.go | 6 +- pkg/common/discoveryregister/etcd/etcd.go | 110 ++++++++++++++++++ 4 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 pkg/common/discoveryregister/etcd/etcd.go diff --git a/go.mod b/go.mod index 3a6ae0bd9..a87a3b6d7 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.18.2 github.com/stathat/consistent v1.0.0 + go.etcd.io/etcd/client/v3 v3.5.13 go.uber.org/automaxprocs v1.5.3 golang.org/x/sync v0.6.0 ) @@ -58,6 +59,8 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/clbanning/mxj v1.8.4 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -137,6 +140,8 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.etcd.io/etcd/api/v3 v3.5.13 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect diff --git a/go.sum b/go.sum index fa72d61a2..5611a6ca6 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -111,6 +115,7 @@ github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= @@ -378,6 +383,12 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4= +go.etcd.io/etcd/api/v3 v3.5.13/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c= +go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg= +go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8= +go.etcd.io/etcd/client/v3 v3.5.13 h1:o0fHTNJLeO0MyVbc7I3fsCf6nrOqn5d+diSarKnB2js= +go.etcd.io/etcd/client/v3 v3.5.13/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI= go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 38d7382fa..4aeee67bc 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -16,6 +16,7 @@ package discoveryregister import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" + getcd "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/discovery/zookeeper" @@ -27,10 +28,11 @@ const ( zookeeperConst = "zookeeper" kubenetesConst = "k8s" directConst = "direct" + etcd = "etcd" ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { +func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry1, error) { switch share.Env { case zookeeperConst: @@ -44,6 +46,8 @@ func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share ) case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) + case etcd: + return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) case directConst: //return direct.NewConnDirect(config) default: diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go new file mode 100644 index 000000000..5c6bae849 --- /dev/null +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -0,0 +1,110 @@ +package etcd + +import ( + "context" + "fmt" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/naming/endpoints" + "go.etcd.io/etcd/client/v3/naming/resolver" + "google.golang.org/grpc" + gresolver "google.golang.org/grpc/resolver" + + "log" + "time" +) + +// SvcDiscoveryRegistryImpl implementation +type SvcDiscoveryRegistryImpl struct { + client *clientv3.Client + resolver gresolver.Builder + dialOptions []grpc.DialOption + serviceKey string + endpointMgr endpoints.Manager + leaseID clientv3.LeaseID + schema string +} + +func NewSvcDiscoveryRegistry(schema string, endpoints []string) (*SvcDiscoveryRegistryImpl, error) { + cfg := clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 5 * time.Second, + } + client, err := clientv3.New(cfg) + if err != nil { + return nil, err + } + r, err := resolver.NewBuilder(client) + if err != nil { + return nil, err + } + return &SvcDiscoveryRegistryImpl{ + client: client, + resolver: r, + schema: schema, + }, nil +} + +func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { + target := fmt.Sprintf("%s:///%s", r.schema, serviceName) + conn, err := grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) + if err != nil { + return nil, err + } + return []*grpc.ClientConn{conn}, nil +} + +func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + target := fmt.Sprintf("%s:///%s", r.schema, serviceName) + return grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) +} + +func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { + return fmt.Sprintf("%s:///%s", r.schema, r.serviceKey) +} + +func (r *SvcDiscoveryRegistryImpl) AddOption(opts ...grpc.DialOption) { + r.dialOptions = append(r.dialOptions, opts...) +} + +func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { + if err := conn.Close(); err != nil { + log.Printf("Failed to close connection: %v", err) + } +} + +func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { + r.serviceKey = fmt.Sprintf("%s/%s:%d", serviceName, host, port) + em, err := endpoints.NewManager(r.client, serviceName) + if err != nil { + return err + } + r.endpointMgr = em + + leaseResp, err := r.client.Grant(context.Background(), 5) + if err != nil { + return err + } + r.leaseID = leaseResp.ID + + endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} + err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) + if err != nil { + return err + } + + _, err = r.client.KeepAlive(context.Background(), r.leaseID) + return err +} + +func (r *SvcDiscoveryRegistryImpl) UnRegister() error { + if r.endpointMgr == nil { + return fmt.Errorf("endpoint manager is not initialized") + } + return r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) +} + +func (r *SvcDiscoveryRegistryImpl) Close() { + if r.client != nil { + _ = r.client.Close() + } +} From 3020661c7ce6794a63c5eaf476dec3355daef490 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:37:43 +0800 Subject: [PATCH 002/143] add etcd --- pkg/common/startrpc/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index ebcd5aa7c..1fdeec402 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -46,7 +46,7 @@ import ( // Start rpc server. func Start[T any](ctx context.Context, zookeeperConfig *config2.ZooKeeper, prometheusConfig *config2.Prometheus, listenIP, registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { + config T, client discovery.SvcDiscoveryRegistry1, server *grpc.Server) error, options ...grpc.ServerOption) error { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) if err != nil { From 3845eb0965a842745ca9fa406d1734fea00c5c9c Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:54:16 +0800 Subject: [PATCH 003/143] add etcd --- pkg/common/discoveryregister/discoveryregister.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 4aeee67bc..da1822f36 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -28,7 +28,7 @@ const ( zookeeperConst = "zookeeper" kubenetesConst = "k8s" directConst = "direct" - etcd = "etcd" + etcdConst = "etcd" ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. @@ -46,7 +46,7 @@ func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share ) case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) - case etcd: + case etcdConst: return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) case directConst: //return direct.NewConnDirect(config) From eef480efb110dd6b855613df0cc681d97e65f038 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:55:54 +0800 Subject: [PATCH 004/143] add etcd --- pkg/common/discoveryregister/discoveryregister.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index da1822f36..0868e6f51 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -33,6 +33,7 @@ const ( // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry1, error) { + return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) switch share.Env { case zookeeperConst: From 21eb693629fe3450c4f8b02fc473fa2e99488c60 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 09:25:08 +0800 Subject: [PATCH 005/143] etcd naming and discovery --- pkg/common/discoveryregister/discoveryregister.go | 4 +--- pkg/common/startrpc/start.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 0868e6f51..31bee9267 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -32,11 +32,9 @@ const ( ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry1, error) { - return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) +func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { switch share.Env { case zookeeperConst: - return zookeeper.NewZkClient( zookeeperConfig.Address, zookeeperConfig.Schema, diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 1fdeec402..ebcd5aa7c 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -46,7 +46,7 @@ import ( // Start rpc server. func Start[T any](ctx context.Context, zookeeperConfig *config2.ZooKeeper, prometheusConfig *config2.Prometheus, listenIP, registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry1, server *grpc.Server) error, options ...grpc.ServerOption) error { + config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) if err != nil { From 95a1e331b755cd66bf92c8d876fdd01d1a649641 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:30:12 +0800 Subject: [PATCH 006/143] add etcd --- go.mod | 2 +- go.sum | 4 ++-- pkg/common/discoveryregister/discoveryregister.go | 7 ++++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a87a3b6d7..6b6de7804 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.19 + github.com/openimsdk/tools v0.0.49-alpha.6 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 5611a6ca6..9a1bd14aa 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.19 h1:CbASL0yefRSVAmWPVeRnhF7wZKd6umLfz31CIhEgrBs= -github.com/openimsdk/tools v0.0.49-alpha.19/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.6 h1:I3HSyNWqOMTzZ1DlD0ZDKAgVzxGAFnN9EYW0xkA+8To= +github.com/openimsdk/tools v0.0.49-alpha.6/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 31bee9267..4aeee67bc 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -28,13 +28,14 @@ const ( zookeeperConst = "zookeeper" kubenetesConst = "k8s" directConst = "direct" - etcdConst = "etcd" + etcd = "etcd" ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { +func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry1, error) { switch share.Env { case zookeeperConst: + return zookeeper.NewZkClient( zookeeperConfig.Address, zookeeperConfig.Schema, @@ -45,7 +46,7 @@ func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share ) case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) - case etcdConst: + case etcd: return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) case directConst: //return direct.NewConnDirect(config) From 22b21f4d2ae9fc7c676ea50f9e28ea1338e14fa6 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:37:43 +0800 Subject: [PATCH 007/143] add etcd --- pkg/common/startrpc/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index ebcd5aa7c..1fdeec402 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -46,7 +46,7 @@ import ( // Start rpc server. func Start[T any](ctx context.Context, zookeeperConfig *config2.ZooKeeper, prometheusConfig *config2.Prometheus, listenIP, registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { + config T, client discovery.SvcDiscoveryRegistry1, server *grpc.Server) error, options ...grpc.ServerOption) error { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) if err != nil { From 78719683ff36a0b3922fff8f1193cea88a5defdd Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:54:16 +0800 Subject: [PATCH 008/143] add etcd --- pkg/common/discoveryregister/discoveryregister.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 4aeee67bc..da1822f36 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -28,7 +28,7 @@ const ( zookeeperConst = "zookeeper" kubenetesConst = "k8s" directConst = "direct" - etcd = "etcd" + etcdConst = "etcd" ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. @@ -46,7 +46,7 @@ func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share ) case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) - case etcd: + case etcdConst: return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) case directConst: //return direct.NewConnDirect(config) From 8382ea3d9da1d9d4ce39397f207b178042546dce Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:55:54 +0800 Subject: [PATCH 009/143] add etcd --- pkg/common/discoveryregister/discoveryregister.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index da1822f36..0868e6f51 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -33,6 +33,7 @@ const ( // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry1, error) { + return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) switch share.Env { case zookeeperConst: From bdd6f8dbdbd1ab9c14f4852743ceec25b6369329 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 09:25:08 +0800 Subject: [PATCH 010/143] add etcd --- go.mod | 2 +- go.sum | 4 ++-- pkg/common/discoveryregister/discoveryregister.go | 4 +--- pkg/common/startrpc/start.go | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 6b6de7804..467b58675 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.6 + github.com/openimsdk/tools v0.0.49-alpha.8 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 9a1bd14aa..fa1b15aa3 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.6 h1:I3HSyNWqOMTzZ1DlD0ZDKAgVzxGAFnN9EYW0xkA+8To= -github.com/openimsdk/tools v0.0.49-alpha.6/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.8 h1:/l7u13eT6PByF21JOYuIteEUa9/XR63VwJWg0EHx3Ew= +github.com/openimsdk/tools v0.0.49-alpha.8/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 0868e6f51..31bee9267 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -32,11 +32,9 @@ const ( ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry1, error) { - return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) +func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { switch share.Env { case zookeeperConst: - return zookeeper.NewZkClient( zookeeperConfig.Address, zookeeperConfig.Schema, diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 1fdeec402..ebcd5aa7c 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -46,7 +46,7 @@ import ( // Start rpc server. func Start[T any](ctx context.Context, zookeeperConfig *config2.ZooKeeper, prometheusConfig *config2.Prometheus, listenIP, registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry1, server *grpc.Server) error, options ...grpc.ServerOption) error { + config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) if err != nil { From de931c8f92fff449075c45cb0516b17709938782 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 09:31:16 +0800 Subject: [PATCH 011/143] add etcd --- go.mod | 2 +- go.sum | 4 ++-- pkg/common/discoveryregister/etcd/etcd.go | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 467b58675..2ec3e20b2 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.8 + github.com/openimsdk/tools v0.0.49-alpha.9 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index fa1b15aa3..edecead94 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.8 h1:/l7u13eT6PByF21JOYuIteEUa9/XR63VwJWg0EHx3Ew= -github.com/openimsdk/tools v0.0.49-alpha.8/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.9 h1:yoa3GS6t0d1mRv/S86niFBGDgSjy2EWWwBI5NAH1Kgk= +github.com/openimsdk/tools v0.0.49-alpha.9/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 5c6bae849..ce76fae15 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -44,6 +44,9 @@ func NewSvcDiscoveryRegistry(schema string, endpoints []string) (*SvcDiscoveryRe }, nil } +func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { + return "", nil +} func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) conn, err := grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) From be188948b9914f9757faaccb64ea2aa0286fe1a1 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 09:50:11 +0800 Subject: [PATCH 012/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index ce76fae15..db27ebf8f 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -83,7 +83,7 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, } r.endpointMgr = em - leaseResp, err := r.client.Grant(context.Background(), 5) + leaseResp, err := r.client.Grant(context.Background(), 30) if err != nil { return err } @@ -91,11 +91,6 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) - if err != nil { - return err - } - - _, err = r.client.KeepAlive(context.Background(), r.leaseID) return err } From 3a706cb061dd5a509d1ad99792825b152d8bbff7 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 09:56:37 +0800 Subject: [PATCH 013/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index db27ebf8f..cb8284e72 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -76,7 +76,7 @@ func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { } func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s:%d", serviceName, host, port) + r.serviceKey = fmt.Sprintf("%s/%s/%d", serviceName, host, port) em, err := endpoints.NewManager(r.client, serviceName) if err != nil { return err From e5b477a9c35f810531840bb53cd801cd8edf9a59 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:09:40 +0800 Subject: [PATCH 014/143] add etcd --- pkg/common/discoveryregister/discoveryregister.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 31bee9267..c2deb6cf2 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -46,7 +46,7 @@ func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) case etcdConst: - return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) + return getcd.NewSvcDiscoveryRegistry("etcd", []string{"http://localhost:2379"}) case directConst: //return direct.NewConnDirect(config) default: From fa6904033817254637693d3496285ca405b35363 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:19:05 +0800 Subject: [PATCH 015/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index cb8284e72..81ec77600 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -49,7 +49,7 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, } func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) - conn, err := grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) + conn, err := grpc.DialContext(ctx, target, grpc.WithResolvers(r.resolver)) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) - return grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) + return grpc.DialContext(ctx, target, grpc.WithResolvers(r.resolver)) } func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { From 28cb3a37fad22e0e7a487ba6db3ac1076cb547b0 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:30:47 +0800 Subject: [PATCH 016/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 81ec77600..871bc07a0 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -49,7 +49,7 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, } func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) - conn, err := grpc.DialContext(ctx, target, grpc.WithResolvers(r.resolver)) + conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) - return grpc.DialContext(ctx, target, grpc.WithResolvers(r.resolver)) + return grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) } func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { From 719ffc7816fdb9d95bd8cf78dff28275fd52c32a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:55:57 +0800 Subject: [PATCH 017/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 871bc07a0..9edb48d82 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -76,7 +76,7 @@ func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { } func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s/%d", serviceName, host, port) + r.serviceKey = fmt.Sprintf("%s/%s-%d", serviceName, host, port) em, err := endpoints.NewManager(r.client, serviceName) if err != nil { return err @@ -92,6 +92,16 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) return err + + lease, _ := r.client.Grant(context.TODO(), 30) + + em, err = endpoints.NewManager(r.client, "foo/bar/my-service") + if err != nil { + return err + } + + err := em.AddEndpoint(context.TODO(), "foo/bar/my-service/e1", endpoints.Endpoint{Addr: "1.2.3.4"}, clientv3.WithLease(lease.ID)) + } func (r *SvcDiscoveryRegistryImpl) UnRegister() error { From 11914cbd3b711a1396c162eb1c86cbd8f56ac24f Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:59:29 +0800 Subject: [PATCH 018/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 9edb48d82..04bd57ca5 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -92,15 +92,15 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) return err - - lease, _ := r.client.Grant(context.TODO(), 30) - - em, err = endpoints.NewManager(r.client, "foo/bar/my-service") - if err != nil { - return err - } - - err := em.AddEndpoint(context.TODO(), "foo/bar/my-service/e1", endpoints.Endpoint{Addr: "1.2.3.4"}, clientv3.WithLease(lease.ID)) + // + //lease, _ := r.client.Grant(context.TODO(), 30) + // + //em, err = endpoints.NewManager(r.client, "foo/bar/my-service") + //if err != nil { + // return err + //} + // + //err := em.AddEndpoint(context.TODO(), "foo/bar/my-service/e1", endpoints.Endpoint{Addr: "1.2.3.4"}, clientv3.WithLease(lease.ID)) } From cc0cb2db644f7ea038415301c064e3cb1887468e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 11:22:49 +0800 Subject: [PATCH 019/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 04bd57ca5..e627dfedb 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -91,17 +91,12 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) - return err - // - //lease, _ := r.client.Grant(context.TODO(), 30) - // - //em, err = endpoints.NewManager(r.client, "foo/bar/my-service") - //if err != nil { - // return err - //} - // - //err := em.AddEndpoint(context.TODO(), "foo/bar/my-service/e1", endpoints.Endpoint{Addr: "1.2.3.4"}, clientv3.WithLease(lease.ID)) + if err != nil { + return err + } + _, kaErr := r.client.KeepAlive(context.Background(), r.leaseID) + return kaErr } func (r *SvcDiscoveryRegistryImpl) UnRegister() error { From eb8dbcbf5fc520aedeb3a8f8469383dcd9454faf Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 11:40:44 +0800 Subject: [PATCH 020/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index e627dfedb..cc94f259a 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -95,8 +95,26 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, return err } - _, kaErr := r.client.KeepAlive(context.Background(), r.leaseID) - return kaErr + go r.keepAliveLease(r.leaseID) + + return nil +} + +func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { + ch, err := r.client.KeepAlive(context.Background(), leaseID) + if err != nil { + log.Printf("Failed to keep lease alive: %v", err) + return + } + + for ka := range ch { + if ka != nil { + fmt.Printf("Received lease keep-alive response: %v", ka) + } else { + fmt.Printf("Lease keep-alive response channel closed") + break + } + } } func (r *SvcDiscoveryRegistryImpl) UnRegister() error { From 1b8d9a4aba792f8058fab3396170338b0255e65a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 11:44:12 +0800 Subject: [PATCH 021/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index cc94f259a..62d6df150 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -109,7 +109,7 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { for ka := range ch { if ka != nil { - fmt.Printf("Received lease keep-alive response: %v", ka) + fmt.Printf("Received lease keep-alive response: %v\n", ka) } else { fmt.Printf("Lease keep-alive response channel closed") break From 2f572cbc8149ae5b9ebb29809553ffca4c3ff0fb Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 12:03:29 +0800 Subject: [PATCH 022/143] add etcd --- config/discovery.yml | 16 ++++++++++++++++ .../discoveryregister/discoveryregister.go | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 config/discovery.yml diff --git a/config/discovery.yml b/config/discovery.yml new file mode 100644 index 000000000..6e0922ede --- /dev/null +++ b/config/discovery.yml @@ -0,0 +1,16 @@ + +enable: "etcd" +etcd: + schema: openim + address: [ http://localhost:2379 ] + username: '' + password: '' + + +zookeeper: + schema: openim + address: [ localhost:12181 ] + username: '' + password: '' + + diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index c2deb6cf2..dbf16eda0 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -46,7 +46,7 @@ func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) case etcdConst: - return getcd.NewSvcDiscoveryRegistry("etcd", []string{"http://localhost:2379"}) + return getcd.NewSvcDiscoveryRegistry("etcd", []string{"localhost:2379"}) case directConst: //return direct.NewConnDirect(config) default: From add7ea8ef015661e37e760d4fdbd0f5e330fda1e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 12:18:18 +0800 Subject: [PATCH 023/143] add etcd --- .env | 2 +- config/discovery.yml | 4 ++-- docker-compose.yml | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.env b/.env index d3ae0426a..1e7b1e11a 100644 --- a/.env +++ b/.env @@ -4,7 +4,7 @@ REDIS_IMAGE=redis:7.0.0 ZOOKEEPER_IMAGE=bitnami/zookeeper:3.8 KAFKA_IMAGE=bitnami/kafka:3.5.1 MINIO_IMAGE=minio/minio:RELEASE.2024-01-11T07-46-16Z - +ETCD_IMAGE=quay.io/coreos/etcd:v3.5.13 OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.5.1 OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.7 diff --git a/config/discovery.yml b/config/discovery.yml index 6e0922ede..2cc54be72 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,12 +1,12 @@ enable: "etcd" etcd: - schema: openim - address: [ http://localhost:2379 ] + address: [ localhost:12379 ] username: '' password: '' + zookeeper: schema: openim address: [ localhost:12181 ] diff --git a/docker-compose.yml b/docker-compose.yml index aeb53a417..d72c1a2fa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,6 +58,26 @@ services: networks: - openim + etcd: + image: "${ETCD_IMAGE}" + container_name: etcd + ports: + - "12379:2379" + - "12380:2380" + environment: + - ETCD_NAME=s1 + - ETCD_DATA_DIR=/etcd-data + - ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379 + - ETCD_ADVERTISE_CLIENT_URLS=http://0.0.0.0:2379 + - ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380 + - ETCD_INITIAL_ADVERTISE_PEER_URLS=http://0.0.0.0:2380 + - ETCD_INITIAL_CLUSTER=s1=http://0.0.0.0:2380 + - ETCD_INITIAL_CLUSTER_TOKEN=tkn + - ETCD_INITIAL_CLUSTER_STATE=new + restart: always + networks: + - openim + kafka: image: "${KAFKA_IMAGE}" container_name: kafka From e94598f5c1e74fb40bba5316f07b409afb5607c3 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 15:40:17 +0800 Subject: [PATCH 024/143] Add etcd as a service discovery mechanism --- config/discovery.yml | 5 +- config/share.yml | 1 - internal/api/init.go | 19 ++-- internal/msggateway/hub_server.go | 2 +- internal/msggateway/init.go | 8 +- internal/msgtransfer/init.go | 3 +- internal/push/push.go | 2 +- internal/rpc/auth/auth.go | 8 +- internal/rpc/conversation/conversaion.go | 9 +- internal/rpc/friend/friend.go | 9 +- internal/rpc/group/group.go | 2 +- internal/rpc/msg/server.go | 2 +- internal/rpc/third/third.go | 2 +- internal/rpc/user/user.go | 2 +- internal/tools/cron_task.go | 8 +- pkg/common/cmd/api.go | 4 +- pkg/common/cmd/auth.go | 4 +- pkg/common/cmd/constant.go | 4 +- pkg/common/cmd/conversation.go | 4 +- pkg/common/cmd/cron_task.go | 2 +- pkg/common/cmd/friend.go | 4 +- pkg/common/cmd/group.go | 4 +- pkg/common/cmd/msg.go | 4 +- pkg/common/cmd/msg_gateway.go | 2 +- pkg/common/cmd/msg_transfer.go | 1 + pkg/common/cmd/push.go | 4 +- pkg/common/cmd/third.go | 4 +- pkg/common/cmd/user.go | 4 +- pkg/common/config/config.go | 14 ++- .../discoveryregister/discoveryregister.go | 34 ++++--- pkg/common/discoveryregister/etcd/etcd.go | 90 ++++++++++++++----- pkg/common/startrpc/start.go | 4 +- 32 files changed, 161 insertions(+), 108 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 2cc54be72..3d96ff9b6 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,16 +1,13 @@ - enable: "etcd" etcd: + rootDirectory: openim address: [ localhost:12379 ] username: '' password: '' - - zookeeper: schema: openim address: [ localhost:12181 ] username: '' password: '' - diff --git a/config/share.yml b/config/share.yml index 2abbb77a0..fc97b6a1f 100644 --- a/config/share.yml +++ b/config/share.yml @@ -1,5 +1,4 @@ secret: openIM123 -env: zookeeper rpcRegisterName: user: user friend: friend diff --git a/internal/api/init.go b/internal/api/init.go index 6e784da9a..b49a14569 100644 --- a/internal/api/init.go +++ b/internal/api/init.go @@ -38,20 +38,17 @@ import ( ) type Config struct { - RpcConfig config.API - MongodbConfig config.Mongo - ZookeeperConfig config.ZooKeeper - NotificationConfig config.Notification - Share config.Share - MinioConfig config.Minio + API config.API + Share config.Share + Discovery config.Discovery } func Start(ctx context.Context, index int, config *Config) error { - apiPort, err := datautil.GetElemByIndex(config.RpcConfig.Api.Ports, index) + apiPort, err := datautil.GetElemByIndex(config.API.Api.Ports, index) if err != nil { return err } - prometheusPort, err := datautil.GetElemByIndex(config.RpcConfig.Prometheus.Ports, index) + prometheusPort, err := datautil.GetElemByIndex(config.API.Prometheus.Ports, index) if err != nil { return err } @@ -59,7 +56,7 @@ func Start(ctx context.Context, index int, config *Config) error { var client discovery.SvcDiscoveryRegistry // Determine whether zk is passed according to whether it is a clustered deployment - client, err = kdisc.NewDiscoveryRegister(&config.ZookeeperConfig, &config.Share) + client, err = kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } @@ -70,7 +67,7 @@ func Start(ctx context.Context, index int, config *Config) error { ) router := newGinRouter(client, config) - if config.RpcConfig.Prometheus.Enable { + if config.API.Prometheus.Enable { go func() { p := ginprom.NewPrometheus("app", prommetrics.GetGinCusMetrics("Api")) p.SetListenAddress(fmt.Sprintf(":%d", prometheusPort)) @@ -81,7 +78,7 @@ func Start(ctx context.Context, index int, config *Config) error { }() } - address := net.JoinHostPort(network.GetListenIP(config.RpcConfig.Api.ListenIP), strconv.Itoa(apiPort)) + address := net.JoinHostPort(network.GetListenIP(config.API.Api.ListenIP), strconv.Itoa(apiPort)) server := http.Server{Addr: address, Handler: router} log.CInfo(ctx, "API server is initializing", "address", address, "apiPort", apiPort, "prometheusPort", prometheusPort) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index bfe81b602..f9bb699ed 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -35,7 +35,7 @@ func (s *Server) InitServer(ctx context.Context, config *Config, disCov discover } func (s *Server) Start(ctx context.Context, index int, conf *Config) error { - return startrpc.Start(ctx, &conf.ZookeeperConfig, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, + return startrpc.Start(ctx, &conf.Discovery, &conf.MsgGateway.Prometheus, conf.MsgGateway.ListenIP, conf.MsgGateway.RPC.RegisterIP, conf.MsgGateway.RPC.Ports, index, conf.Share.RpcRegisterName.MessageGateway, diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 727ade0af..ef24d1bf9 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -24,10 +24,10 @@ import ( ) type Config struct { - MsgGateway config.MsgGateway - ZookeeperConfig config.ZooKeeper - Share config.Share - WebhooksConfig config.Webhooks + MsgGateway config.MsgGateway + Share config.Share + WebhooksConfig config.Webhooks + Discovery config.Discovery } // Start run ws server. diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 68d953e90..8f72e979d 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -63,6 +63,7 @@ type Config struct { ZookeeperConfig config.ZooKeeper Share config.Share WebhooksConfig config.Webhooks + Discovery config.Discovery } func Start(ctx context.Context, index int, config *Config) error { @@ -76,7 +77,7 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - client, err := kdisc.NewDiscoveryRegister(&config.ZookeeperConfig, &config.Share) + client, err := kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) if err != nil { return err } diff --git a/internal/push/push.go b/internal/push/push.go index 18012a864..2e5c4e526 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -24,11 +24,11 @@ type Config struct { RedisConfig config.Redis MongodbConfig config.Mongo KafkaConfig config.Kafka - ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks LocalCacheConfig config.LocalCache + Discovery config.Discovery } func (p pushServer) PushMsg(ctx context.Context, req *pbpush.PushMsgReq) (*pbpush.PushMsgResp, error) { diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index c6d236b21..ddb655398 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -45,10 +45,10 @@ type authServer struct { } type Config struct { - RpcConfig config.Auth - RedisConfig config.Redis - ZookeeperConfig config.ZooKeeper - Share config.Share + RpcConfig config.Auth + RedisConfig config.Redis + Share config.Share + Discovery config.Discovery } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 96d2a403f..ec7522212 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -48,13 +48,14 @@ type conversationServer struct { } type Config struct { - RpcConfig config.Conversation - RedisConfig config.Redis - MongodbConfig config.Mongo - ZookeeperConfig config.ZooKeeper + RpcConfig config.Conversation + RedisConfig config.Redis + MongodbConfig config.Mongo + // ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share LocalCacheConfig config.LocalCache + Discovery config.Discovery } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { diff --git a/internal/rpc/friend/friend.go b/internal/rpc/friend/friend.go index bffda3c04..b49490f26 100644 --- a/internal/rpc/friend/friend.go +++ b/internal/rpc/friend/friend.go @@ -50,14 +50,15 @@ type friendServer struct { } type Config struct { - RpcConfig config.Friend - RedisConfig config.Redis - MongodbConfig config.Mongo - ZookeeperConfig config.ZooKeeper + RpcConfig config.Friend + RedisConfig config.Redis + MongodbConfig config.Mongo + //ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks LocalCacheConfig config.LocalCache + Discovery config.Discovery } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 13bd7f9be..551554c23 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -68,11 +68,11 @@ type Config struct { RpcConfig config.Group RedisConfig config.Redis MongodbConfig config.Mongo - ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks LocalCacheConfig config.LocalCache + Discovery config.Discovery } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { diff --git a/internal/rpc/msg/server.go b/internal/rpc/msg/server.go index 3f4df8d4b..5d7c0b297 100644 --- a/internal/rpc/msg/server.go +++ b/internal/rpc/msg/server.go @@ -59,11 +59,11 @@ type ( RedisConfig config.Redis MongodbConfig config.Mongo KafkaConfig config.Kafka - ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks LocalCacheConfig config.LocalCache + Discovery config.Discovery } ) diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 9bf8cafa9..a3d9085d3 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -46,11 +46,11 @@ type Config struct { RpcConfig config.Third RedisConfig config.Redis MongodbConfig config.Mongo - ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share MinioConfig config.Minio LocalCacheConfig config.LocalCache + Discovery config.Discovery } func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error { diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index c453ac9f8..a28fa24e2 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -61,11 +61,11 @@ type Config struct { RedisConfig config.Redis MongodbConfig config.Mongo KafkaConfig config.Kafka - ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share WebhooksConfig config.Webhooks LocalCacheConfig config.LocalCache + Discovery config.Discovery } func Start(ctx context.Context, config *Config, client registry.SvcDiscoveryRegistry, server *grpc.Server) error { diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 20baeffaf..bf037b694 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -33,9 +33,9 @@ import ( ) type CronTaskConfig struct { - CronTask config.CronTask - ZookeeperConfig config.ZooKeeper - Share config.Share + CronTask config.CronTask + Share config.Share + Discovery config.Discovery } func Start(ctx context.Context, config *CronTaskConfig) error { @@ -43,7 +43,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { if config.CronTask.RetainChatRecords < 1 { return errs.New("msg destruct time must be greater than 1").Wrap() } - client, err := kdisc.NewDiscoveryRegister(&config.ZookeeperConfig, &config.Share) + client, err := kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) if err != nil { return errs.WrapMsg(err, "failed to register discovery service") } diff --git a/pkg/common/cmd/api.go b/pkg/common/cmd/api.go index 022fb1097..ecdb0dd3a 100644 --- a/pkg/common/cmd/api.go +++ b/pkg/common/cmd/api.go @@ -33,9 +33,9 @@ func NewApiCmd() *ApiCmd { var apiConfig api.Config ret := &ApiCmd{apiConfig: &apiConfig} ret.configMap = map[string]any{ - OpenIMAPICfgFileName: &apiConfig.RpcConfig, - ZookeeperConfigFileName: &apiConfig.ZookeeperConfig, + OpenIMAPICfgFileName: &apiConfig.API, ShareFileName: &apiConfig.Share, + DiscoveryConfigFilename: &apiConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) diff --git a/pkg/common/cmd/auth.go b/pkg/common/cmd/auth.go index 5ed02ffd0..7d75a7da6 100644 --- a/pkg/common/cmd/auth.go +++ b/pkg/common/cmd/auth.go @@ -36,8 +36,8 @@ func NewAuthRpcCmd() *AuthRpcCmd { ret.configMap = map[string]any{ OpenIMRPCAuthCfgFileName: &authConfig.RpcConfig, RedisConfigFileName: &authConfig.RedisConfig, - ZookeeperConfigFileName: &authConfig.ZookeeperConfig, ShareFileName: &authConfig.Share, + DiscoveryConfigFilename: &authConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) @@ -53,7 +53,7 @@ func (a *AuthRpcCmd) Exec() error { } func (a *AuthRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.authConfig.ZookeeperConfig, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, + return startrpc.Start(a.ctx, &a.authConfig.Discovery, &a.authConfig.RpcConfig.Prometheus, a.authConfig.RpcConfig.RPC.ListenIP, a.authConfig.RpcConfig.RPC.RegisterIP, a.authConfig.RpcConfig.RPC.Ports, a.Index(), a.authConfig.Share.RpcRegisterName.Auth, &a.authConfig.Share, a.authConfig, auth.Start) } diff --git a/pkg/common/cmd/constant.go b/pkg/common/cmd/constant.go index 55eb4a069..d7c7ee94e 100644 --- a/pkg/common/cmd/constant.go +++ b/pkg/common/cmd/constant.go @@ -42,6 +42,7 @@ var ( OpenIMRPCMsgCfgFileName string OpenIMRPCThirdCfgFileName string OpenIMRPCUserCfgFileName string + DiscoveryConfigFilename string ) var ConfigEnvPrefixMap map[string]string @@ -70,6 +71,7 @@ func init() { OpenIMRPCMsgCfgFileName = "openim-rpc-msg.yml" OpenIMRPCThirdCfgFileName = "openim-rpc-third.yml" OpenIMRPCUserCfgFileName = "openim-rpc-user.yml" + DiscoveryConfigFilename = "discovery.yml" ConfigEnvPrefixMap = make(map[string]string) fileNames := []string{ @@ -79,7 +81,7 @@ func init() { OpenIMAPICfgFileName, OpenIMCronTaskCfgFileName, OpenIMMsgGatewayCfgFileName, OpenIMMsgTransferCfgFileName, OpenIMPushCfgFileName, OpenIMRPCAuthCfgFileName, OpenIMRPCConversationCfgFileName, OpenIMRPCFriendCfgFileName, OpenIMRPCGroupCfgFileName, - OpenIMRPCMsgCfgFileName, OpenIMRPCThirdCfgFileName, OpenIMRPCUserCfgFileName, + OpenIMRPCMsgCfgFileName, OpenIMRPCThirdCfgFileName, OpenIMRPCUserCfgFileName, DiscoveryConfigFilename, } for _, fileName := range fileNames { diff --git a/pkg/common/cmd/conversation.go b/pkg/common/cmd/conversation.go index 0a617c729..57ffa52bc 100644 --- a/pkg/common/cmd/conversation.go +++ b/pkg/common/cmd/conversation.go @@ -36,11 +36,11 @@ func NewConversationRpcCmd() *ConversationRpcCmd { ret.configMap = map[string]any{ OpenIMRPCConversationCfgFileName: &conversationConfig.RpcConfig, RedisConfigFileName: &conversationConfig.RedisConfig, - ZookeeperConfigFileName: &conversationConfig.ZookeeperConfig, MongodbConfigFileName: &conversationConfig.MongodbConfig, ShareFileName: &conversationConfig.Share, NotificationFileName: &conversationConfig.NotificationConfig, LocalCacheConfigFileName: &conversationConfig.LocalCacheConfig, + DiscoveryConfigFilename: &conversationConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) @@ -55,7 +55,7 @@ func (a *ConversationRpcCmd) Exec() error { } func (a *ConversationRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.conversationConfig.ZookeeperConfig, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, + return startrpc.Start(a.ctx, &a.conversationConfig.Discovery, &a.conversationConfig.RpcConfig.Prometheus, a.conversationConfig.RpcConfig.RPC.ListenIP, a.conversationConfig.RpcConfig.RPC.RegisterIP, a.conversationConfig.RpcConfig.RPC.Ports, a.Index(), a.conversationConfig.Share.RpcRegisterName.Conversation, &a.conversationConfig.Share, a.conversationConfig, conversation.Start) } diff --git a/pkg/common/cmd/cron_task.go b/pkg/common/cmd/cron_task.go index be26f5af3..fd4447524 100644 --- a/pkg/common/cmd/cron_task.go +++ b/pkg/common/cmd/cron_task.go @@ -34,8 +34,8 @@ func NewCronTaskCmd() *CronTaskCmd { ret := &CronTaskCmd{cronTaskConfig: &cronTaskConfig} ret.configMap = map[string]any{ OpenIMCronTaskCfgFileName: &cronTaskConfig.CronTask, - ZookeeperConfigFileName: &cronTaskConfig.ZookeeperConfig, ShareFileName: &cronTaskConfig.Share, + DiscoveryConfigFilename: &cronTaskConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) diff --git a/pkg/common/cmd/friend.go b/pkg/common/cmd/friend.go index b8d46f77e..8be1f7745 100644 --- a/pkg/common/cmd/friend.go +++ b/pkg/common/cmd/friend.go @@ -36,12 +36,12 @@ func NewFriendRpcCmd() *FriendRpcCmd { ret.configMap = map[string]any{ OpenIMRPCFriendCfgFileName: &friendConfig.RpcConfig, RedisConfigFileName: &friendConfig.RedisConfig, - ZookeeperConfigFileName: &friendConfig.ZookeeperConfig, MongodbConfigFileName: &friendConfig.MongodbConfig, ShareFileName: &friendConfig.Share, NotificationFileName: &friendConfig.NotificationConfig, WebhooksConfigFileName: &friendConfig.WebhooksConfig, LocalCacheConfigFileName: &friendConfig.LocalCacheConfig, + DiscoveryConfigFilename: &friendConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) @@ -56,7 +56,7 @@ func (a *FriendRpcCmd) Exec() error { } func (a *FriendRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.friendConfig.ZookeeperConfig, &a.friendConfig.RpcConfig.Prometheus, a.friendConfig.RpcConfig.RPC.ListenIP, + return startrpc.Start(a.ctx, &a.friendConfig.Discovery, &a.friendConfig.RpcConfig.Prometheus, a.friendConfig.RpcConfig.RPC.ListenIP, a.friendConfig.RpcConfig.RPC.RegisterIP, a.friendConfig.RpcConfig.RPC.Ports, a.Index(), a.friendConfig.Share.RpcRegisterName.Friend, &a.friendConfig.Share, a.friendConfig, friend.Start) } diff --git a/pkg/common/cmd/group.go b/pkg/common/cmd/group.go index 8bf977824..f158b8c62 100644 --- a/pkg/common/cmd/group.go +++ b/pkg/common/cmd/group.go @@ -36,12 +36,12 @@ func NewGroupRpcCmd() *GroupRpcCmd { ret.configMap = map[string]any{ OpenIMRPCGroupCfgFileName: &groupConfig.RpcConfig, RedisConfigFileName: &groupConfig.RedisConfig, - ZookeeperConfigFileName: &groupConfig.ZookeeperConfig, MongodbConfigFileName: &groupConfig.MongodbConfig, ShareFileName: &groupConfig.Share, NotificationFileName: &groupConfig.NotificationConfig, WebhooksConfigFileName: &groupConfig.WebhooksConfig, LocalCacheConfigFileName: &groupConfig.LocalCacheConfig, + DiscoveryConfigFilename: &groupConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) @@ -56,7 +56,7 @@ func (a *GroupRpcCmd) Exec() error { } func (a *GroupRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.groupConfig.ZookeeperConfig, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, + return startrpc.Start(a.ctx, &a.groupConfig.Discovery, &a.groupConfig.RpcConfig.Prometheus, a.groupConfig.RpcConfig.RPC.ListenIP, a.groupConfig.RpcConfig.RPC.RegisterIP, a.groupConfig.RpcConfig.RPC.Ports, a.Index(), a.groupConfig.Share.RpcRegisterName.Group, &a.groupConfig.Share, a.groupConfig, group.Start) } diff --git a/pkg/common/cmd/msg.go b/pkg/common/cmd/msg.go index a3b521b4b..91f7931fb 100644 --- a/pkg/common/cmd/msg.go +++ b/pkg/common/cmd/msg.go @@ -36,13 +36,13 @@ func NewMsgRpcCmd() *MsgRpcCmd { ret.configMap = map[string]any{ OpenIMRPCMsgCfgFileName: &msgConfig.RpcConfig, RedisConfigFileName: &msgConfig.RedisConfig, - ZookeeperConfigFileName: &msgConfig.ZookeeperConfig, MongodbConfigFileName: &msgConfig.MongodbConfig, KafkaConfigFileName: &msgConfig.KafkaConfig, ShareFileName: &msgConfig.Share, NotificationFileName: &msgConfig.NotificationConfig, WebhooksConfigFileName: &msgConfig.WebhooksConfig, LocalCacheConfigFileName: &msgConfig.LocalCacheConfig, + DiscoveryConfigFilename: &msgConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) @@ -57,7 +57,7 @@ func (a *MsgRpcCmd) Exec() error { } func (a *MsgRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.msgConfig.ZookeeperConfig, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, + return startrpc.Start(a.ctx, &a.msgConfig.Discovery, &a.msgConfig.RpcConfig.Prometheus, a.msgConfig.RpcConfig.RPC.ListenIP, a.msgConfig.RpcConfig.RPC.RegisterIP, a.msgConfig.RpcConfig.RPC.Ports, a.Index(), a.msgConfig.Share.RpcRegisterName.Msg, &a.msgConfig.Share, a.msgConfig, msg.Start) } diff --git a/pkg/common/cmd/msg_gateway.go b/pkg/common/cmd/msg_gateway.go index 897fd7008..78004094c 100644 --- a/pkg/common/cmd/msg_gateway.go +++ b/pkg/common/cmd/msg_gateway.go @@ -36,9 +36,9 @@ func NewMsgGatewayCmd() *MsgGatewayCmd { ret := &MsgGatewayCmd{msgGatewayConfig: &msgGatewayConfig} ret.configMap = map[string]any{ OpenIMMsgGatewayCfgFileName: &msgGatewayConfig.MsgGateway, - ZookeeperConfigFileName: &msgGatewayConfig.ZookeeperConfig, ShareFileName: &msgGatewayConfig.Share, WebhooksConfigFileName: &msgGatewayConfig.WebhooksConfig, + DiscoveryConfigFilename: &msgGatewayConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) diff --git a/pkg/common/cmd/msg_transfer.go b/pkg/common/cmd/msg_transfer.go index 86f42dc56..99acc053c 100644 --- a/pkg/common/cmd/msg_transfer.go +++ b/pkg/common/cmd/msg_transfer.go @@ -40,6 +40,7 @@ func NewMsgTransferCmd() *MsgTransferCmd { ZookeeperConfigFileName: &msgTransferConfig.ZookeeperConfig, ShareFileName: &msgTransferConfig.Share, WebhooksConfigFileName: &msgTransferConfig.WebhooksConfig, + DiscoveryConfigFilename: &msgTransferConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index 0140ced23..3e7c4c249 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -36,13 +36,13 @@ func NewPushRpcCmd() *PushRpcCmd { ret.configMap = map[string]any{ OpenIMPushCfgFileName: &pushConfig.RpcConfig, RedisConfigFileName: &pushConfig.RedisConfig, - ZookeeperConfigFileName: &pushConfig.ZookeeperConfig, MongodbConfigFileName: &pushConfig.MongodbConfig, KafkaConfigFileName: &pushConfig.KafkaConfig, ShareFileName: &pushConfig.Share, NotificationFileName: &pushConfig.NotificationConfig, WebhooksConfigFileName: &pushConfig.WebhooksConfig, LocalCacheConfigFileName: &pushConfig.LocalCacheConfig, + DiscoveryConfigFilename: &pushConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) @@ -57,7 +57,7 @@ func (a *PushRpcCmd) Exec() error { } func (a *PushRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.pushConfig.ZookeeperConfig, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, + return startrpc.Start(a.ctx, &a.pushConfig.Discovery, &a.pushConfig.RpcConfig.Prometheus, a.pushConfig.RpcConfig.RPC.ListenIP, a.pushConfig.RpcConfig.RPC.RegisterIP, a.pushConfig.RpcConfig.RPC.Ports, a.Index(), a.pushConfig.Share.RpcRegisterName.Push, &a.pushConfig.Share, a.pushConfig, push.Start) } diff --git a/pkg/common/cmd/third.go b/pkg/common/cmd/third.go index 0dfa7d5be..b6731f1ff 100644 --- a/pkg/common/cmd/third.go +++ b/pkg/common/cmd/third.go @@ -36,12 +36,12 @@ func NewThirdRpcCmd() *ThirdRpcCmd { ret.configMap = map[string]any{ OpenIMRPCThirdCfgFileName: &thirdConfig.RpcConfig, RedisConfigFileName: &thirdConfig.RedisConfig, - ZookeeperConfigFileName: &thirdConfig.ZookeeperConfig, MongodbConfigFileName: &thirdConfig.MongodbConfig, ShareFileName: &thirdConfig.Share, NotificationFileName: &thirdConfig.NotificationConfig, MinioConfigFileName: &thirdConfig.MinioConfig, LocalCacheConfigFileName: &thirdConfig.LocalCacheConfig, + DiscoveryConfigFilename: &thirdConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) @@ -56,7 +56,7 @@ func (a *ThirdRpcCmd) Exec() error { } func (a *ThirdRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.thirdConfig.ZookeeperConfig, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, + return startrpc.Start(a.ctx, &a.thirdConfig.Discovery, &a.thirdConfig.RpcConfig.Prometheus, a.thirdConfig.RpcConfig.RPC.ListenIP, a.thirdConfig.RpcConfig.RPC.RegisterIP, a.thirdConfig.RpcConfig.RPC.Ports, a.Index(), a.thirdConfig.Share.RpcRegisterName.Third, &a.thirdConfig.Share, a.thirdConfig, third.Start) } diff --git a/pkg/common/cmd/user.go b/pkg/common/cmd/user.go index 315b93256..674f9e3a6 100644 --- a/pkg/common/cmd/user.go +++ b/pkg/common/cmd/user.go @@ -36,13 +36,13 @@ func NewUserRpcCmd() *UserRpcCmd { ret.configMap = map[string]any{ OpenIMRPCUserCfgFileName: &userConfig.RpcConfig, RedisConfigFileName: &userConfig.RedisConfig, - ZookeeperConfigFileName: &userConfig.ZookeeperConfig, MongodbConfigFileName: &userConfig.MongodbConfig, KafkaConfigFileName: &userConfig.KafkaConfig, ShareFileName: &userConfig.Share, NotificationFileName: &userConfig.NotificationConfig, WebhooksConfigFileName: &userConfig.WebhooksConfig, LocalCacheConfigFileName: &userConfig.LocalCacheConfig, + DiscoveryConfigFilename: &userConfig.Discovery, } ret.RootCmd = NewRootCmd(program.GetProcessName(), WithConfigMap(ret.configMap)) ret.ctx = context.WithValue(context.Background(), "version", config.Version) @@ -57,7 +57,7 @@ func (a *UserRpcCmd) Exec() error { } func (a *UserRpcCmd) runE() error { - return startrpc.Start(a.ctx, &a.userConfig.ZookeeperConfig, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, + return startrpc.Start(a.ctx, &a.userConfig.Discovery, &a.userConfig.RpcConfig.Prometheus, a.userConfig.RpcConfig.RPC.ListenIP, a.userConfig.RpcConfig.RPC.RegisterIP, a.userConfig.RpcConfig.RPC.Ports, a.Index(), a.userConfig.Share.RpcRegisterName.User, &a.userConfig.Share, a.userConfig, user.Start) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 24d04d8cc..12c4f7f78 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -345,7 +345,6 @@ type AfterConfig struct { type Share struct { Secret string `mapstructure:"secret"` - Env string `mapstructure:"env"` RpcRegisterName RpcRegisterName `mapstructure:"rpcRegisterName"` IMAdminUserID []string `mapstructure:"imAdminUserID"` } @@ -432,6 +431,19 @@ type ZooKeeper struct { Password string `mapstructure:"password"` } +type Discovery struct { + Enable string `mapstructure:"enable"` + Etcd Etcd `mapstructure:"etcd"` + ZooKeeper ZooKeeper `mapstructure:"zooKeeper"` +} + +type Etcd struct { + RootDirectory string `mapstructure:"rootDirectory"` + Address []string `mapstructure:"address"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` +} + func (m *Mongo) Build() *mongoutil.Config { return &mongoutil.Config{ Uri: m.URI, diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index dbf16eda0..1085ec1ea 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -24,33 +24,31 @@ import ( "time" ) -const ( - zookeeperConst = "zookeeper" - kubenetesConst = "k8s" - directConst = "direct" - etcdConst = "etcd" -) - // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { - switch share.Env { - case zookeeperConst: +func NewDiscoveryRegister(discovery *config.Discovery, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { + switch discovery.Enable { + case "zookeeper": return zookeeper.NewZkClient( - zookeeperConfig.Address, - zookeeperConfig.Schema, + discovery.ZooKeeper.Address, + discovery.ZooKeeper.Schema, zookeeper.WithFreq(time.Hour), - zookeeper.WithUserNameAndPassword(zookeeperConfig.Username, zookeeperConfig.Password), + zookeeper.WithUserNameAndPassword(discovery.ZooKeeper.Username, discovery.ZooKeeper.Password), zookeeper.WithRoundRobin(), zookeeper.WithTimeout(10), ) - case kubenetesConst: + case "k8s": return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) - case etcdConst: - return getcd.NewSvcDiscoveryRegistry("etcd", []string{"localhost:2379"}) - case directConst: + case "etcd": + return getcd.NewSvcDiscoveryRegistry( + discovery.Etcd.RootDirectory, + discovery.Etcd.Address, + getcd.WithDialTimeout(10*time.Second), + getcd.WithMaxCallSendMsgSize(20*1024*1024), + getcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) + case "direct": //return direct.NewConnDirect(config) default: - return nil, errs.New("unsupported discovery type", "type", share.Env).Wrap() + return nil, errs.New("unsupported discovery type", "type", discovery.Enable).Wrap() } return nil, nil } diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 62d6df150..904f83dd0 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -8,27 +8,38 @@ import ( "go.etcd.io/etcd/client/v3/naming/resolver" "google.golang.org/grpc" gresolver "google.golang.org/grpc/resolver" - - "log" "time" ) +// ZkOption defines a function type for modifying clientv3.Config +type ZkOption func(*clientv3.Config) + // SvcDiscoveryRegistryImpl implementation type SvcDiscoveryRegistryImpl struct { - client *clientv3.Client - resolver gresolver.Builder - dialOptions []grpc.DialOption - serviceKey string - endpointMgr endpoints.Manager - leaseID clientv3.LeaseID - schema string + client *clientv3.Client + resolver gresolver.Builder + dialOptions []grpc.DialOption + serviceKey string + endpointMgr endpoints.Manager + leaseID clientv3.LeaseID + rootDirectory string } -func NewSvcDiscoveryRegistry(schema string, endpoints []string) (*SvcDiscoveryRegistryImpl, error) { +// NewSvcDiscoveryRegistry creates a new service discovery registry implementation +func NewSvcDiscoveryRegistry(rootDirectory string, endpoints []string, options ...ZkOption) (*SvcDiscoveryRegistryImpl, error) { cfg := clientv3.Config{ Endpoints: endpoints, DialTimeout: 5 * time.Second, + // Increase keep-alive queue capacity and message size + PermitWithoutStream: true, + MaxCallSendMsgSize: 10 * 1024 * 1024, // 10 MB } + + // Apply provided options to the config + for _, opt := range options { + opt(&cfg) + } + client, err := clientv3.New(cfg) if err != nil { return nil, err @@ -38,17 +49,42 @@ func NewSvcDiscoveryRegistry(schema string, endpoints []string) (*SvcDiscoveryRe return nil, err } return &SvcDiscoveryRegistryImpl{ - client: client, - resolver: r, - schema: schema, + client: client, + resolver: r, + rootDirectory: rootDirectory, }, nil } +// WithDialTimeout sets a custom dial timeout for the etcd client +func WithDialTimeout(timeout time.Duration) ZkOption { + return func(cfg *clientv3.Config) { + cfg.DialTimeout = timeout + } +} + +// WithMaxCallSendMsgSize sets a custom max call send message size for the etcd client +func WithMaxCallSendMsgSize(size int) ZkOption { + return func(cfg *clientv3.Config) { + cfg.MaxCallSendMsgSize = size + } +} + +// WithUsernameAndPassword sets a username and password for the etcd client +func WithUsernameAndPassword(username, password string) ZkOption { + return func(cfg *clientv3.Config) { + cfg.Username = username + cfg.Password = password + } +} + +// GetUserIdHashGatewayHost returns the gateway host for a given user ID hash func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { return "", nil } + +// GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - target := fmt.Sprintf("%s:///%s", r.schema, serviceName) + target := fmt.Sprintf("etcd:///%s", serviceName) conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { return nil, err @@ -56,34 +92,39 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str return []*grpc.ClientConn{conn}, nil } +// GetConn returns a single gRPC client connection for a given service name func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - target := fmt.Sprintf("%s:///%s", r.schema, serviceName) + target := fmt.Sprintf("etcd:///%s", serviceName) return grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) } +// GetSelfConnTarget returns the connection target for the current service func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { - return fmt.Sprintf("%s:///%s", r.schema, r.serviceKey) + return fmt.Sprintf("etcd:///%s", r.serviceKey) } +// AddOption appends gRPC dial options to the existing options func (r *SvcDiscoveryRegistryImpl) AddOption(opts ...grpc.DialOption) { r.dialOptions = append(r.dialOptions, opts...) } +// CloseConn closes a given gRPC client connection func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { if err := conn.Close(); err != nil { - log.Printf("Failed to close connection: %v", err) + fmt.Printf("Failed to close connection: %v\n", err) } } +// Register registers a new service endpoint with etcd func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s-%d", serviceName, host, port) - em, err := endpoints.NewManager(r.client, serviceName) + r.serviceKey = fmt.Sprintf("%s/%s/%s-%d", r.rootDirectory, serviceName, host, port) + em, err := endpoints.NewManager(r.client, r.rootDirectory+"/"+serviceName) if err != nil { return err } r.endpointMgr = em - leaseResp, err := r.client.Grant(context.Background(), 30) + leaseResp, err := r.client.Grant(context.Background(), 60) // Increase TTL time if err != nil { return err } @@ -100,10 +141,11 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, return nil } +// keepAliveLease maintains the lease alive by sending keep-alive requests func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { ch, err := r.client.KeepAlive(context.Background(), leaseID) if err != nil { - log.Printf("Failed to keep lease alive: %v", err) + fmt.Printf("Failed to keep lease alive: %v\n", err) return } @@ -111,12 +153,13 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { if ka != nil { fmt.Printf("Received lease keep-alive response: %v\n", ka) } else { - fmt.Printf("Lease keep-alive response channel closed") - break + fmt.Printf("Lease keep-alive response channel closed\n") + return } } } +// UnRegister removes the service endpoint from etcd func (r *SvcDiscoveryRegistryImpl) UnRegister() error { if r.endpointMgr == nil { return fmt.Errorf("endpoint manager is not initialized") @@ -124,6 +167,7 @@ func (r *SvcDiscoveryRegistryImpl) UnRegister() error { return r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) } +// Close closes the etcd client connection func (r *SvcDiscoveryRegistryImpl) Close() { if r.client != nil { _ = r.client.Close() diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index ebcd5aa7c..a36bcfe1c 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -44,7 +44,7 @@ import ( ) // Start rpc server. -func Start[T any](ctx context.Context, zookeeperConfig *config2.ZooKeeper, prometheusConfig *config2.Prometheus, listenIP, +func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusConfig *config2.Prometheus, listenIP, registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { @@ -68,7 +68,7 @@ func Start[T any](ctx context.Context, zookeeperConfig *config2.ZooKeeper, prome } defer listener.Close() - client, err := kdisc.NewDiscoveryRegister(zookeeperConfig, share) + client, err := kdisc.NewDiscoveryRegister(discovery, share) if err != nil { return err } From 140f734728ae2bff03ce6f7820f9879c32bec7a4 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 15:51:57 +0800 Subject: [PATCH 025/143] Add etcd as a service discovery mechanism --- internal/api/router.go | 2 +- internal/msggateway/n_ws_server.go | 2 +- internal/push/onlinepusher.go | 13 +++++-------- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index bd2de99db..1fbb33b09 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -34,7 +34,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En messageRpc := rpcclient.NewMessage(disCov, config.Share.RpcRegisterName.Msg) conversationRpc := rpcclient.NewConversation(disCov, config.Share.RpcRegisterName.Conversation) authRpc := rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) - thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.RpcConfig.Prometheus.GrafanaURL) + thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.API.Prometheus.GrafanaURL) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index cf607d470..e601962f5 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -267,7 +267,7 @@ func (ws *WsServer) registerClient(client *Client) { } wg := sync.WaitGroup{} - if ws.msgGatewayConfig.Share.Env == "zookeeper" { + if ws.msgGatewayConfig.Discovery.Enable != "k8s" { wg.Add(1) go func() { defer wg.Done() diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 30bdf3e2e..9c9189d37 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -12,11 +12,6 @@ import ( "sync" ) -const ( - KUBERNETES = "k8s" - ZOOKEEPER = "zookeeper" -) - type OnlinePusher interface { GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) @@ -42,10 +37,12 @@ func (u emptyOnlinePUsher) GetOnlinePushFailedUserIDs(ctx context.Context, msg * } func NewOnlinePusher(disCov discovery.SvcDiscoveryRegistry, config *Config) OnlinePusher { - switch config.Share.Env { - case KUBERNETES: + switch config.Discovery.Enable { + case "k8s": return NewK8sStaticConsistentHash(disCov, config) - case ZOOKEEPER: + case "zookeeper": + return NewDefaultAllNode(disCov, config) + case "etcd": return NewDefaultAllNode(disCov, config) default: return newEmptyOnlinePUsher() From 7ece310c93896f3d3864ac722b2e64088ff49760 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 16:01:16 +0800 Subject: [PATCH 026/143] Add etcd as a service discovery mechanism --- config/zookeeper.yml | 6 --- internal/rpc/conversation/conversaion.go | 7 ++- pkg/common/cmd/constant.go | 4 +- .../discoveryregister/zookeeper/zookeeper.go | 44 ------------------- 4 files changed, 4 insertions(+), 57 deletions(-) delete mode 100644 config/zookeeper.yml delete mode 100644 pkg/common/discoveryregister/zookeeper/zookeeper.go diff --git a/config/zookeeper.yml b/config/zookeeper.yml deleted file mode 100644 index 33f52d7ca..000000000 --- a/config/zookeeper.yml +++ /dev/null @@ -1,6 +0,0 @@ - -schema: openim -address: [ localhost:12181 ] -username: '' -password: '' - diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index ec7522212..4c7828610 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -48,10 +48,9 @@ type conversationServer struct { } type Config struct { - RpcConfig config.Conversation - RedisConfig config.Redis - MongodbConfig config.Mongo - // ZookeeperConfig config.ZooKeeper + RpcConfig config.Conversation + RedisConfig config.Redis + MongodbConfig config.Mongo NotificationConfig config.Notification Share config.Share LocalCacheConfig config.LocalCache diff --git a/pkg/common/cmd/constant.go b/pkg/common/cmd/constant.go index d7c7ee94e..45dbcafda 100644 --- a/pkg/common/cmd/constant.go +++ b/pkg/common/cmd/constant.go @@ -26,7 +26,6 @@ var ( LocalCacheConfigFileName string KafkaConfigFileName string RedisConfigFileName string - ZookeeperConfigFileName string MongodbConfigFileName string MinioConfigFileName string LogConfigFileName string @@ -55,7 +54,6 @@ func init() { LocalCacheConfigFileName = "local-cache.yml" KafkaConfigFileName = "kafka.yml" RedisConfigFileName = "redis.yml" - ZookeeperConfigFileName = "zookeeper.yml" MongodbConfigFileName = "mongodb.yml" MinioConfigFileName = "minio.yml" LogConfigFileName = "log.yml" @@ -76,7 +74,7 @@ func init() { ConfigEnvPrefixMap = make(map[string]string) fileNames := []string{ FileName, NotificationFileName, ShareFileName, WebhooksConfigFileName, - KafkaConfigFileName, RedisConfigFileName, ZookeeperConfigFileName, + KafkaConfigFileName, RedisConfigFileName, MongodbConfigFileName, MinioConfigFileName, LogConfigFileName, OpenIMAPICfgFileName, OpenIMCronTaskCfgFileName, OpenIMMsgGatewayCfgFileName, OpenIMMsgTransferCfgFileName, OpenIMPushCfgFileName, OpenIMRPCAuthCfgFileName, diff --git a/pkg/common/discoveryregister/zookeeper/zookeeper.go b/pkg/common/discoveryregister/zookeeper/zookeeper.go deleted file mode 100644 index 1d11414b6..000000000 --- a/pkg/common/discoveryregister/zookeeper/zookeeper.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package zookeeper - -import ( - "os" - "strings" -) - -// getEnv returns the value of an environment variable if it exists, otherwise it returns the fallback value. -func getEnv(key, fallback string) string { - if value, exists := os.LookupEnv(key); exists { - return value - } - return fallback -} - -// getZkAddrFromEnv returns the Zookeeper addresses combined from the ZOOKEEPER_ADDRESS and ZOOKEEPER_PORT environment variables. -// If the environment variables are not set, it returns the fallback value. -func getZkAddrFromEnv(fallback []string) []string { - address, addrExists := os.LookupEnv("ZOOKEEPER_ADDRESS") - port, portExists := os.LookupEnv("ZOOKEEPER_PORT") - - if addrExists && portExists { - addresses := strings.Split(address, ",") - for i, addr := range addresses { - addresses[i] = addr + ":" + port - } - return addresses - } - return fallback -} From 764088a5d0983cc9a96943eb6f146908623e2491 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 16:03:51 +0800 Subject: [PATCH 027/143] Add etcd as a service discovery mechanism --- internal/msgtransfer/init.go | 15 +++++++-------- pkg/common/cmd/msg_transfer.go | 1 - 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 8f72e979d..3384b8493 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -56,14 +56,13 @@ type MsgTransfer struct { } type Config struct { - MsgTransfer config.MsgTransfer - RedisConfig config.Redis - MongodbConfig config.Mongo - KafkaConfig config.Kafka - ZookeeperConfig config.ZooKeeper - Share config.Share - WebhooksConfig config.Webhooks - Discovery config.Discovery + MsgTransfer config.MsgTransfer + RedisConfig config.Redis + MongodbConfig config.Mongo + KafkaConfig config.Kafka + Share config.Share + WebhooksConfig config.Webhooks + Discovery config.Discovery } func Start(ctx context.Context, index int, config *Config) error { diff --git a/pkg/common/cmd/msg_transfer.go b/pkg/common/cmd/msg_transfer.go index 99acc053c..0d48281e5 100644 --- a/pkg/common/cmd/msg_transfer.go +++ b/pkg/common/cmd/msg_transfer.go @@ -37,7 +37,6 @@ func NewMsgTransferCmd() *MsgTransferCmd { RedisConfigFileName: &msgTransferConfig.RedisConfig, MongodbConfigFileName: &msgTransferConfig.MongodbConfig, KafkaConfigFileName: &msgTransferConfig.KafkaConfig, - ZookeeperConfigFileName: &msgTransferConfig.ZookeeperConfig, ShareFileName: &msgTransferConfig.Share, WebhooksConfigFileName: &msgTransferConfig.WebhooksConfig, DiscoveryConfigFilename: &msgTransferConfig.Discovery, From a6bb70a366ca11a8888323961ef975cd56dad539 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 16:44:15 +0800 Subject: [PATCH 028/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 46 ++++++++++++++++++++ tools/check-component/main.go | 53 +++++++++++++++++------ 2 files changed, 85 insertions(+), 14 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 904f83dd0..51fafd0b6 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -3,6 +3,7 @@ package etcd import ( "context" "fmt" + "github.com/pkg/errors" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/naming/endpoints" "go.etcd.io/etcd/client/v3/naming/resolver" @@ -173,3 +174,48 @@ func (r *SvcDiscoveryRegistryImpl) Close() { _ = r.client.Close() } } + +// Check verifies if etcd is running by checking the existence of the root node and optionally creates it +func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfNotExist bool, options ...ZkOption) error { + // Configure the etcd client with default settings + cfg := clientv3.Config{ + Endpoints: etcdServers, + } + + // Apply provided options to the config + for _, opt := range options { + opt(&cfg) + } + + client, err := clientv3.New(cfg) + if err != nil { + return errors.Wrap(err, "failed to connect to etcd") + } + defer client.Close() + + // Create a child context with a default timeout or use the provided context + opCtx, cancel := context.WithCancel(ctx) + defer cancel() + + // Check if the root node exists + resp, err := client.Get(opCtx, etcdRoot) + if err != nil { + return errors.Wrap(err, "failed to get the root node from etcd") + } + + // If root node does not exist and createIfNotExist is true, create the root node + if len(resp.Kvs) == 0 { + if createIfNotExist { + _, err := client.Put(opCtx, etcdRoot, "") + if err != nil { + return errors.Wrap(err, "failed to create the root node in etcd") + } + fmt.Printf("Root node %s did not exist, but has been created.\n", etcdRoot) + } else { + return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) + } + } else { + fmt.Printf("Etcd is running and the root node %s exists.\n", etcdRoot) + } + return nil +} diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 7fe64d3c5..5a936f775 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/discovery/zookeeper" @@ -43,6 +44,25 @@ func CheckZookeeper(ctx context.Context, config *config.ZooKeeper) error { return zookeeper.Check(ctx, config.Address, config.Schema, zookeeper.WithUserNameAndPassword(config.Username, config.Password)) } +func CheckEtcd(ctx context.Context, config *config.Etcd) error { + etcd.Check(ctx, config.Address, "/check_openim_component", + true, + etcd.WithDialTimeout(10*time.Second), + etcd.WithMaxCallSendMsgSize(20*1024*1024), + etcd.WithUsernameAndPassword(config.Username, config.Password)) + return nil +} + +func CheckDiscovery(ctx context.Context, config *config.Discovery) error { + switch config.Enable { + case "etcd": + return CheckEtcd(ctx, &config.Etcd) + + } + + return nil +} + func CheckMongo(ctx context.Context, config *config.Mongo) error { return mongoutil.Check(ctx, config.Build()) } @@ -59,14 +79,14 @@ func CheckKafka(ctx context.Context, conf *config.Kafka) error { return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic}) } -func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, *config.Minio, *config.ZooKeeper, error) { +func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, *config.Minio, *config.Discovery, error) { var ( - mongoConfig = &config.Mongo{} - redisConfig = &config.Redis{} - kafkaConfig = &config.Kafka{} - minioConfig = &config.Minio{} - zookeeperConfig = &config.ZooKeeper{} - thirdConfig = &config.Third{} + mongoConfig = &config.Mongo{} + redisConfig = &config.Redis{} + kafkaConfig = &config.Kafka{} + minioConfig = &config.Minio{} + discovery = &config.Discovery{} + thirdConfig = &config.Third{} ) err := config.LoadConfig(filepath.Join(configDir, cmd.MongodbConfigFileName), cmd.ConfigEnvPrefixMap[cmd.MongodbConfigFileName], mongoConfig) if err != nil { @@ -96,11 +116,11 @@ func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, } else { minioConfig = nil } - err = config.LoadConfig(filepath.Join(configDir, cmd.ZookeeperConfigFileName), cmd.ConfigEnvPrefixMap[cmd.ZookeeperConfigFileName], zookeeperConfig) + err = config.LoadConfig(filepath.Join(configDir, cmd.DiscoveryConfigFilename), cmd.ConfigEnvPrefixMap[cmd.DiscoveryConfigFilename], discovery) if err != nil { return nil, nil, nil, nil, nil, err } - return mongoConfig, redisConfig, kafkaConfig, minioConfig, zookeeperConfig, nil + return mongoConfig, redisConfig, kafkaConfig, minioConfig, discovery, nil } func main() { @@ -127,13 +147,10 @@ func main() { } } -func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig *config.Redis, kafkaConfig *config.Kafka, minioConfig *config.Minio, zookeeperConfig *config.ZooKeeper, maxRetry int) error { +func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig *config.Redis, kafkaConfig *config.Kafka, minioConfig *config.Minio, discovery *config.Discovery, maxRetry int) error { checksDone := make(map[string]bool) checks := map[string]func() error{ - "Zookeeper": func() error { - return CheckZookeeper(ctx, zookeeperConfig) - }, "Mongo": func() error { return CheckMongo(ctx, mongoConfig) }, @@ -144,12 +161,20 @@ func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig * return CheckKafka(ctx, kafkaConfig) }, } - if minioConfig != nil { checks["MinIO"] = func() error { return CheckMinIO(ctx, minioConfig) } } + if discovery.Enable == "etcd" { + checks["Etcd"] = func() error { + return CheckEtcd(ctx, &discovery.Etcd) + } + } else if discovery.Enable == "zookeeper" { + checks["Zookeeper"] = func() error { + return CheckZookeeper(ctx, &discovery.ZooKeeper) + } + } for i := 0; i < maxRetry; i++ { allSuccess := true From 6b1f7a44074a81710193a0ada4093375c73f1fea Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 16:58:49 +0800 Subject: [PATCH 029/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 12 +++++++----- tools/check-component/main.go | 13 +------------ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 51fafd0b6..eb7e6ac8b 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -193,12 +193,14 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN } defer client.Close() - // Create a child context with a default timeout or use the provided context - opCtx, cancel := context.WithCancel(ctx) - defer cancel() + if cfg.DialTimeout != 0 { + ctx, _ = context.WithTimeout(ctx, cfg.DialTimeout) + } else { + ctx, _ = context.WithTimeout(ctx, 10*time.Second) + } // Check if the root node exists - resp, err := client.Get(opCtx, etcdRoot) + resp, err := client.Get(ctx, etcdRoot) if err != nil { return errors.Wrap(err, "failed to get the root node from etcd") } @@ -206,7 +208,7 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN // If root node does not exist and createIfNotExist is true, create the root node if len(resp.Kvs) == 0 { if createIfNotExist { - _, err := client.Put(opCtx, etcdRoot, "") + _, err := client.Put(ctx, etcdRoot, "") if err != nil { return errors.Wrap(err, "failed to create the root node in etcd") } diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 5a936f775..5d4435e26 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -45,22 +45,11 @@ func CheckZookeeper(ctx context.Context, config *config.ZooKeeper) error { } func CheckEtcd(ctx context.Context, config *config.Etcd) error { - etcd.Check(ctx, config.Address, "/check_openim_component", + return etcd.Check(ctx, config.Address, "/check_openim_component", true, etcd.WithDialTimeout(10*time.Second), etcd.WithMaxCallSendMsgSize(20*1024*1024), etcd.WithUsernameAndPassword(config.Username, config.Password)) - return nil -} - -func CheckDiscovery(ctx context.Context, config *config.Discovery) error { - switch config.Enable { - case "etcd": - return CheckEtcd(ctx, &config.Etcd) - - } - - return nil } func CheckMongo(ctx context.Context, config *config.Mongo) error { From bc305f910e51ec7c05366a8ae5d0404d7ebcbcab Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 17:01:35 +0800 Subject: [PATCH 030/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index eb7e6ac8b..6866186b0 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -212,7 +212,6 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN if err != nil { return errors.Wrap(err, "failed to create the root node in etcd") } - fmt.Printf("Root node %s did not exist, but has been created.\n", etcdRoot) } else { return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) } From 61dd8a6eeb27ddd810884e8cdafc6c9d129dcdda Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 17:02:38 +0800 Subject: [PATCH 031/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 6866186b0..1f952bbf7 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -152,7 +152,6 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { for ka := range ch { if ka != nil { - fmt.Printf("Received lease keep-alive response: %v\n", ka) } else { fmt.Printf("Lease keep-alive response channel closed\n") return From 063ba0b0a166a10854077deabd2b4b675f4aef63 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 17:19:00 +0800 Subject: [PATCH 032/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 34 +++++++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 1f952bbf7..d9eb86aa3 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -174,7 +174,7 @@ func (r *SvcDiscoveryRegistryImpl) Close() { } } -// Check verifies if etcd is running by checking the existence of the root node and optionally creates it +// Check verifies if etcd is running by checking the existence of the root node and optionally creates it with a lease func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfNotExist bool, options ...ZkOption) error { // Configure the etcd client with default settings cfg := clientv3.Config{ @@ -192,30 +192,52 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN } defer client.Close() + // Determine timeout for context + var opCtx context.Context + var cancel context.CancelFunc if cfg.DialTimeout != 0 { - ctx, _ = context.WithTimeout(ctx, cfg.DialTimeout) + opCtx, cancel = context.WithTimeout(ctx, cfg.DialTimeout) } else { - ctx, _ = context.WithTimeout(ctx, 10*time.Second) + opCtx, cancel = context.WithTimeout(ctx, 10*time.Second) } + defer cancel() // Check if the root node exists - resp, err := client.Get(ctx, etcdRoot) + resp, err := client.Get(opCtx, etcdRoot) if err != nil { return errors.Wrap(err, "failed to get the root node from etcd") } - // If root node does not exist and createIfNotExist is true, create the root node + // If root node does not exist and createIfNotExist is true, create the root node with a lease if len(resp.Kvs) == 0 { if createIfNotExist { - _, err := client.Put(ctx, etcdRoot, "") + var leaseTTL int64 = 10 + var leaseResp *clientv3.LeaseGrantResponse + if leaseTTL > 0 { + // Create a lease + leaseResp, err = client.Grant(opCtx, leaseTTL) + if err != nil { + return errors.Wrap(err, "failed to create lease in etcd") + } + } + + // Put the key with the lease + putOpts := []clientv3.OpOption{} + if leaseResp != nil { + putOpts = append(putOpts, clientv3.WithLease(leaseResp.ID)) + } + + _, err := client.Put(opCtx, etcdRoot, "", putOpts...) if err != nil { return errors.Wrap(err, "failed to create the root node in etcd") } + fmt.Printf("Root node %s did not exist, but has been created.\n", etcdRoot) } else { return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) } } else { fmt.Printf("Etcd is running and the root node %s exists.\n", etcdRoot) } + return nil } From fa6a2f8ae39cbfe330bffd711780a15cd5d91469 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 17:39:15 +0800 Subject: [PATCH 033/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index d9eb86aa3..741ceb8b1 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -85,7 +85,7 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, // GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - target := fmt.Sprintf("etcd:///%s", serviceName) + target := fmt.Sprintf("etcd:///%s/%s", r.rootDirectory, serviceName) conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { return nil, err @@ -95,7 +95,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str // GetConn returns a single gRPC client connection for a given service name func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - target := fmt.Sprintf("etcd:///%s", serviceName) + target := fmt.Sprintf("etcd:///%s/%s", r.rootDirectory, serviceName) return grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) } From d71700a287c46497c0f5776279186985c1d93fc8 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 18:06:54 +0800 Subject: [PATCH 034/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 741ceb8b1..1707bf776 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -17,12 +17,14 @@ type ZkOption func(*clientv3.Config) // SvcDiscoveryRegistryImpl implementation type SvcDiscoveryRegistryImpl struct { - client *clientv3.Client - resolver gresolver.Builder - dialOptions []grpc.DialOption - serviceKey string - endpointMgr endpoints.Manager - leaseID clientv3.LeaseID + client *clientv3.Client + resolver gresolver.Builder + dialOptions []grpc.DialOption + serviceKey string + endpointMgr endpoints.Manager + leaseID clientv3.LeaseID + rpcRegisterTarget string + rootDirectory string } @@ -101,7 +103,9 @@ func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName stri // GetSelfConnTarget returns the connection target for the current service func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { - return fmt.Sprintf("etcd:///%s", r.serviceKey) + return r.rpcRegisterTarget + // return fmt.Sprintf("etcd:///%s", r.serviceKey) + } // AddOption appends gRPC dial options to the existing options @@ -131,7 +135,9 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, } r.leaseID = leaseResp.ID - endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} + r.rpcRegisterTarget = fmt.Sprintf("%s:%d", host, port) + endpoint := endpoints.Endpoint{Addr: r.rpcRegisterTarget} + err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) if err != nil { return err From ec052e26852cfb9a9c393bde32f9fc34c98caad6 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 18:09:32 +0800 Subject: [PATCH 035/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index e601962f5..fc9f88d59 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,7 +216,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - + log.ZDebug(ctx, "target ", v.Target()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) _, err := msgClient.MultiTerminalLoginCheck(ctx, &msggateway.MultiTerminalLoginCheckReq{ From ccc178ae78e7180f19b62bebc041e72d54b6a2fb Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 18:21:26 +0800 Subject: [PATCH 036/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index fc9f88d59..3ed5e48a6 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,7 +216,9 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } + fmt.Printf("Connection target: %s\n", v.Target()) log.ZDebug(ctx, "target ", v.Target()) + wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) _, err := msgClient.MultiTerminalLoginCheck(ctx, &msggateway.MultiTerminalLoginCheckReq{ From 2c6070a543e2b8b626cb8c62733362a6bfae0158 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 18:26:47 +0800 Subject: [PATCH 037/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 3ed5e48a6..2e38c1720 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,8 +216,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - fmt.Printf("Connection target: %s\n", v.Target()) - log.ZDebug(ctx, "target ", v.Target()) + log.ZDebug(ctx, "conn ", "target", v.Target()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) From ab71bbd7b82ddcc24aec6187a1328a077e3893a4 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 10:06:17 +0800 Subject: [PATCH 038/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 2e38c1720..749fa40b5 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,7 +216,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - log.ZDebug(ctx, "conn ", "target", v.Target()) + log.ZDebug(ctx, "sendUserOnlineInfoToOtherNode conn ", "target", v.Target(), v.GetState()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) From a298e929fbe40fdf7a6e20bfbba1a87fdbf7e800 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 10:06:36 +0800 Subject: [PATCH 039/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 749fa40b5..6ef2c3213 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,7 +216,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - log.ZDebug(ctx, "sendUserOnlineInfoToOtherNode conn ", "target", v.Target(), v.GetState()) + log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target(), v.GetState()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) From 17a3012fa7e1f58b886eeb4306f773f15e77785a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 10:19:24 +0800 Subject: [PATCH 040/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 6ef2c3213..29cbb51d8 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -211,12 +211,11 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C // Online push user online message to other node for _, v := range conns { - v := v // safe closure var + log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) if v.Target() == ws.disCov.GetSelfConnTarget() { log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target(), v.GetState()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) From 2eaf869f1121f63b3ce4e8cd01c931fd93ac4b1e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 10:22:41 +0800 Subject: [PATCH 041/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 29cbb51d8..a29b58528 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -212,8 +212,11 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C // Online push user online message to other node for _, v := range conns { log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) - if v.Target() == ws.disCov.GetSelfConnTarget() { - log.ZDebug(ctx, "Filter out this node", "node", v.Target()) + //if v.Target() == ws.disCov.GetSelfConnTarget() { + // log.ZDebug(ctx, "Filter out this node", "node", v.Target()) + // continue + //} + if true { continue } From 88e064baf0cfd66e853c29bf762e65cf48b8bac4 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 11:10:25 +0800 Subject: [PATCH 042/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 1707bf776..1950223a1 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -87,12 +87,27 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, // GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - target := fmt.Sprintf("etcd:///%s/%s", r.rootDirectory, serviceName) - conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) + var conns []*grpc.ClientConn + // Construct the full key for the service + fullServiceKey := fmt.Sprintf("%s/%s", r.rootDirectory, serviceName) + + // List all endpoints for the service + resp, err := r.client.Get(ctx, fullServiceKey, clientv3.WithPrefix()) if err != nil { return nil, err } - return []*grpc.ClientConn{conn}, nil + + for _, kv := range resp.Kvs { + endpoint := string(kv.Key[len(fullServiceKey)+1:]) // Extract the endpoint address + target := fmt.Sprintf("etcd://%s", endpoint) + conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) + if err != nil { + return nil, err + } + conns = append(conns, conn) + } + + return conns, nil } // GetConn returns a single gRPC client connection for a given service name From 9e7c676c449dfae8ae0654d61236905cb44d9ddf Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 11:10:42 +0800 Subject: [PATCH 043/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index a29b58528..29cbb51d8 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -212,11 +212,8 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C // Online push user online message to other node for _, v := range conns { log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) - //if v.Target() == ws.disCov.GetSelfConnTarget() { - // log.ZDebug(ctx, "Filter out this node", "node", v.Target()) - // continue - //} - if true { + if v.Target() == ws.disCov.GetSelfConnTarget() { + log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } From 2d15c40fbf85fc5fd40762739dabb204715b0784 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 11:50:01 +0800 Subject: [PATCH 044/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 29cbb51d8..515e1f7e8 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -267,6 +267,8 @@ func (ws *WsServer) registerClient(client *Client) { } wg := sync.WaitGroup{} + log.ZDebug(client.ctx, "ws.msgGatewayConfig.Discovery.Enable", ws.msgGatewayConfig.Discovery.Enable) + if ws.msgGatewayConfig.Discovery.Enable != "k8s" { wg.Add(1) go func() { From e6c2bb0b6ec1895e3c7e80ad5e05269886de5aa9 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 12:15:35 +0800 Subject: [PATCH 045/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 1950223a1..af827e4b7 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -94,6 +94,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str // List all endpoints for the service resp, err := r.client.Get(ctx, fullServiceKey, clientv3.WithPrefix()) if err != nil { + fmt.Println("GetConns get ", fullServiceKey, err.Error()) return nil, err } @@ -102,6 +103,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str target := fmt.Sprintf("etcd://%s", endpoint) conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { + fmt.Println("DialContext ", target, err.Error()) return nil, err } conns = append(conns, conn) From bb0da645a7947e4cc553bdba2d701b196b6f2fd8 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 12:17:56 +0800 Subject: [PATCH 046/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index af827e4b7..5d0e37d0f 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -100,7 +100,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str for _, kv := range resp.Kvs { endpoint := string(kv.Key[len(fullServiceKey)+1:]) // Extract the endpoint address - target := fmt.Sprintf("etcd://%s", endpoint) + target := fmt.Sprintf("etcd://%s/%s/%s", r.rootDirectory, serviceName, endpoint) conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { fmt.Println("DialContext ", target, err.Error()) From d786d6984c80f378acf50a29d844dd7259d03d54 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 14:08:35 +0800 Subject: [PATCH 047/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 5d0e37d0f..bf510051f 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -108,7 +108,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str } conns = append(conns, conn) } - + fmt.Println("GetConns number ", len(conns)) return conns, nil } From 3ab01a56f457c49582e5e6716786f2148732ba00 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 15:04:59 +0800 Subject: [PATCH 048/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 1 + internal/push/onlinepusher.go | 1 + pkg/common/discoveryregister/etcd/etcd.go | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 515e1f7e8..defec16df 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -211,6 +211,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C // Online push user online message to other node for _, v := range conns { + v := v log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) if v.Target() == ws.disCov.GetSelfConnTarget() { log.ZDebug(ctx, "Filter out this node", "node", v.Target()) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 9c9189d37..74e3e5306 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -81,6 +81,7 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M // Online push message for _, conn := range conns { + log.ZDebug(ctx, "get gateway conn detail ", "conn ", *conn) conn := conn // loop var safe wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(conn) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index bf510051f..7baac73f3 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -107,8 +107,9 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str return nil, err } conns = append(conns, conn) + fmt.Println("GetConns detail ", *conn) + } - fmt.Println("GetConns number ", len(conns)) return conns, nil } From 274e51ccab5c18fcf5d36019015729b1d5364b1e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 15:07:40 +0800 Subject: [PATCH 049/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 74e3e5306..ddc500806 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -2,6 +2,7 @@ package push import ( "context" + "fmt" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" @@ -81,7 +82,8 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M // Online push message for _, conn := range conns { - log.ZDebug(ctx, "get gateway conn detail ", "conn ", *conn) + fmt.Println(ctx, "get gateway conn detail ", "conn ", *conn) + conn := conn // loop var safe wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(conn) From 07e1b9c728bbd7e7764eb330589fadcb59b6226b Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 15:22:17 +0800 Subject: [PATCH 050/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 7baac73f3..2c4385f68 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -100,7 +100,8 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str for _, kv := range resp.Kvs { endpoint := string(kv.Key[len(fullServiceKey)+1:]) // Extract the endpoint address - target := fmt.Sprintf("etcd://%s/%s/%s", r.rootDirectory, serviceName, endpoint) + //target := fmt.Sprintf("etcd://%s/%s/%s", r.rootDirectory, serviceName, endpoint) + target := endpoint conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { fmt.Println("DialContext ", target, err.Error()) @@ -140,7 +141,7 @@ func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { // Register registers a new service endpoint with etcd func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s/%s-%d", r.rootDirectory, serviceName, host, port) + r.serviceKey = fmt.Sprintf("%s/%s/%s:%d", r.rootDirectory, serviceName, host, port) em, err := endpoints.NewManager(r.client, r.rootDirectory+"/"+serviceName) if err != nil { return err From 3bc4477fa6da1e2bba6be94349fd4b9bf187fede Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 17:20:15 +0800 Subject: [PATCH 051/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 105 ++++++++++++++-------- 1 file changed, 69 insertions(+), 36 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 2c4385f68..7c2bcff66 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -9,6 +9,8 @@ import ( "go.etcd.io/etcd/client/v3/naming/resolver" "google.golang.org/grpc" gresolver "google.golang.org/grpc/resolver" + "strings" + "sync" "time" ) @@ -26,6 +28,9 @@ type SvcDiscoveryRegistryImpl struct { rpcRegisterTarget string rootDirectory string + + mu sync.RWMutex + connMap map[string][]*grpc.ClientConn } // NewSvcDiscoveryRegistry creates a new service discovery registry implementation @@ -51,11 +56,15 @@ func NewSvcDiscoveryRegistry(rootDirectory string, endpoints []string, options . if err != nil { return nil, err } - return &SvcDiscoveryRegistryImpl{ + + s := &SvcDiscoveryRegistryImpl{ client: client, resolver: r, rootDirectory: rootDirectory, - }, nil + connMap: make(map[string][]*grpc.ClientConn), + } + go s.watchServiceChanges() + return s, nil } // WithDialTimeout sets a custom dial timeout for the etcd client @@ -87,31 +96,11 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, // GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - var conns []*grpc.ClientConn - // Construct the full key for the service fullServiceKey := fmt.Sprintf("%s/%s", r.rootDirectory, serviceName) - // List all endpoints for the service - resp, err := r.client.Get(ctx, fullServiceKey, clientv3.WithPrefix()) - if err != nil { - fmt.Println("GetConns get ", fullServiceKey, err.Error()) - return nil, err - } - - for _, kv := range resp.Kvs { - endpoint := string(kv.Key[len(fullServiceKey)+1:]) // Extract the endpoint address - //target := fmt.Sprintf("etcd://%s/%s/%s", r.rootDirectory, serviceName, endpoint) - target := endpoint - conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) - if err != nil { - fmt.Println("DialContext ", target, err.Error()) - return nil, err - } - conns = append(conns, conn) - fmt.Println("GetConns detail ", *conn) - - } - return conns, nil + r.mu.RLock() + defer r.mu.RUnlock() + return r.connMap[fullServiceKey], nil } // GetConn returns a single gRPC client connection for a given service name @@ -123,8 +112,6 @@ func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName stri // GetSelfConnTarget returns the connection target for the current service func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { return r.rpcRegisterTarget - // return fmt.Sprintf("etcd:///%s", r.serviceKey) - } // AddOption appends gRPC dial options to the existing options @@ -163,7 +150,6 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, } go r.keepAliveLease(r.leaseID) - return nil } @@ -184,12 +170,63 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { } } +// watchServiceChanges watches for changes in the service directory +func (r *SvcDiscoveryRegistryImpl) watchServiceChanges() { + watchChan := r.client.Watch(context.Background(), r.rootDirectory, clientv3.WithPrefix()) + for watchResp := range watchChan { + for _, event := range watchResp.Events { + prefix, _ := r.splitEndpoint(string(event.Kv.Key)) + fmt.Printf("Change detected for prefix: %s\n", prefix) + r.refreshConnMap(prefix) + } + } +} + +// refreshConnMap fetches the latest endpoints and updates the local map +func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { + r.mu.Lock() + defer r.mu.Unlock() + + fullPrefix := fmt.Sprintf("%s/", prefix) + resp, err := r.client.Get(context.Background(), fullPrefix, clientv3.WithPrefix()) + if err != nil { + fmt.Printf("Failed to get endpoints: %v\n", err) + return + } + + // Update the connMap with new connections + for _, kv := range resp.Kvs { + _, addr := r.splitEndpoint(string(kv.Key)) + conn, err := grpc.DialContext(context.Background(), addr, append(r.dialOptions, grpc.WithResolvers(r.resolver))...) + if err != nil { + fmt.Printf("Failed to dial new endpoint: %v\n", err) + continue + } + r.connMap[prefix] = append(r.connMap[prefix], conn) + } +} + +// splitEndpoint splits the endpoint string into prefix and address +func (r *SvcDiscoveryRegistryImpl) splitEndpoint(input string) (string, string) { + lastSlashIndex := strings.LastIndex(input, "/") + if lastSlashIndex != -1 { + part1 := input[:lastSlashIndex] + part2 := input[lastSlashIndex+1:] + return part1, part2 + } + return input, "" +} + // UnRegister removes the service endpoint from etcd func (r *SvcDiscoveryRegistryImpl) UnRegister() error { if r.endpointMgr == nil { return fmt.Errorf("endpoint manager is not initialized") } - return r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) + err := r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) + if err != nil { + return err + } + return nil } // Close closes the etcd client connection @@ -197,16 +234,17 @@ func (r *SvcDiscoveryRegistryImpl) Close() { if r.client != nil { _ = r.client.Close() } + + r.mu.Lock() + defer r.mu.Unlock() } // Check verifies if etcd is running by checking the existence of the root node and optionally creates it with a lease func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfNotExist bool, options ...ZkOption) error { - // Configure the etcd client with default settings cfg := clientv3.Config{ Endpoints: etcdServers, } - // Apply provided options to the config for _, opt := range options { opt(&cfg) } @@ -217,7 +255,6 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN } defer client.Close() - // Determine timeout for context var opCtx context.Context var cancel context.CancelFunc if cfg.DialTimeout != 0 { @@ -227,26 +264,22 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN } defer cancel() - // Check if the root node exists resp, err := client.Get(opCtx, etcdRoot) if err != nil { return errors.Wrap(err, "failed to get the root node from etcd") } - // If root node does not exist and createIfNotExist is true, create the root node with a lease if len(resp.Kvs) == 0 { if createIfNotExist { var leaseTTL int64 = 10 var leaseResp *clientv3.LeaseGrantResponse if leaseTTL > 0 { - // Create a lease leaseResp, err = client.Grant(opCtx, leaseTTL) if err != nil { return errors.Wrap(err, "failed to create lease in etcd") } } - // Put the key with the lease putOpts := []clientv3.OpOption{} if leaseResp != nil { putOpts = append(putOpts, clientv3.WithLease(leaseResp.ID)) From 86a940568080e077769c379122be618149ba5770 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 17:44:40 +0800 Subject: [PATCH 052/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 7c2bcff66..4ce60dbd5 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -193,8 +193,7 @@ func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { fmt.Printf("Failed to get endpoints: %v\n", err) return } - - // Update the connMap with new connections + r.connMap[prefix] = []*grpc.ClientConn{} // Update the connMap with new connections for _, kv := range resp.Kvs { _, addr := r.splitEndpoint(string(kv.Key)) conn, err := grpc.DialContext(context.Background(), addr, append(r.dialOptions, grpc.WithResolvers(r.resolver))...) From 09961d5af1e9936a1dcbd486c278455219e51857 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 17:50:40 +0800 Subject: [PATCH 053/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 4ce60dbd5..247161f9b 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -100,6 +100,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str r.mu.RLock() defer r.mu.RUnlock() + fmt.Printf("all conns ", serviceName, r.connMap[fullServiceKey]) return r.connMap[fullServiceKey], nil } From 188f27b0f651c9324bbeea6a298062122c414574 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 18:06:23 +0800 Subject: [PATCH 054/143] Add etcd as a service discovery mechanism --- internal/msggateway/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index af869dd85..0581a025b 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -286,6 +286,7 @@ func (c *Client) KickOnlineMessage() error { resp := Resp{ ReqIdentifier: WSKickOnlineMsg, } + log.ZDebug(c.ctx, "KickOnlineMessage debug ") err := c.writeBinaryMsg(resp) c.close() return err From e58d8c40a4926c8ec3a840683f30b1b3edeb8537 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 08:57:55 +0800 Subject: [PATCH 055/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 3 --- pkg/common/discoveryregister/etcd/etcd.go | 28 +++++++---------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index ddc500806..9c9189d37 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -2,7 +2,6 @@ package push import ( "context" - "fmt" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" @@ -82,8 +81,6 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M // Online push message for _, conn := range conns { - fmt.Println(ctx, "get gateway conn detail ", "conn ", *conn) - conn := conn // loop var safe wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(conn) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 247161f9b..dd8806378 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -97,10 +97,8 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, // GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { fullServiceKey := fmt.Sprintf("%s/%s", r.rootDirectory, serviceName) - r.mu.RLock() defer r.mu.RUnlock() - fmt.Printf("all conns ", serviceName, r.connMap[fullServiceKey]) return r.connMap[fullServiceKey], nil } @@ -122,9 +120,7 @@ func (r *SvcDiscoveryRegistryImpl) AddOption(opts ...grpc.DialOption) { // CloseConn closes a given gRPC client connection func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { - if err := conn.Close(); err != nil { - fmt.Printf("Failed to close connection: %v\n", err) - } + conn.Close() } // Register registers a new service endpoint with etcd @@ -136,7 +132,7 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, } r.endpointMgr = em - leaseResp, err := r.client.Grant(context.Background(), 60) // Increase TTL time + leaseResp, err := r.client.Grant(context.Background(), 30) // if err != nil { return err } @@ -158,14 +154,11 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { ch, err := r.client.KeepAlive(context.Background(), leaseID) if err != nil { - fmt.Printf("Failed to keep lease alive: %v\n", err) return } - for ka := range ch { if ka != nil { } else { - fmt.Printf("Lease keep-alive response channel closed\n") return } } @@ -175,10 +168,14 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { func (r *SvcDiscoveryRegistryImpl) watchServiceChanges() { watchChan := r.client.Watch(context.Background(), r.rootDirectory, clientv3.WithPrefix()) for watchResp := range watchChan { + updatedPrefixes := make(map[string]struct{}) // Create a set to track updated prefixes + for _, event := range watchResp.Events { prefix, _ := r.splitEndpoint(string(event.Kv.Key)) - fmt.Printf("Change detected for prefix: %s\n", prefix) - r.refreshConnMap(prefix) + if _, alreadyUpdated := updatedPrefixes[prefix]; !alreadyUpdated { + updatedPrefixes[prefix] = struct{}{} // Mark this prefix as updated + r.refreshConnMap(prefix) + } } } } @@ -191,7 +188,6 @@ func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { fullPrefix := fmt.Sprintf("%s/", prefix) resp, err := r.client.Get(context.Background(), fullPrefix, clientv3.WithPrefix()) if err != nil { - fmt.Printf("Failed to get endpoints: %v\n", err) return } r.connMap[prefix] = []*grpc.ClientConn{} // Update the connMap with new connections @@ -199,7 +195,6 @@ func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { _, addr := r.splitEndpoint(string(kv.Key)) conn, err := grpc.DialContext(context.Background(), addr, append(r.dialOptions, grpc.WithResolvers(r.resolver))...) if err != nil { - fmt.Printf("Failed to dial new endpoint: %v\n", err) continue } r.connMap[prefix] = append(r.connMap[prefix], conn) @@ -244,11 +239,9 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN cfg := clientv3.Config{ Endpoints: etcdServers, } - for _, opt := range options { opt(&cfg) } - client, err := clientv3.New(cfg) if err != nil { return errors.Wrap(err, "failed to connect to etcd") @@ -279,7 +272,6 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN return errors.Wrap(err, "failed to create lease in etcd") } } - putOpts := []clientv3.OpOption{} if leaseResp != nil { putOpts = append(putOpts, clientv3.WithLease(leaseResp.ID)) @@ -289,13 +281,9 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN if err != nil { return errors.Wrap(err, "failed to create the root node in etcd") } - fmt.Printf("Root node %s did not exist, but has been created.\n", etcdRoot) } else { return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) } - } else { - fmt.Printf("Etcd is running and the root node %s exists.\n", etcdRoot) } - return nil } From f63c124f75d67f5b34e7fb0a4f5efb9a10d929cc Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 10:42:24 +0800 Subject: [PATCH 056/143] Add etcd as a service discovery mechanism --- go.mod | 4 +- go.sum | 4 +- .../discoveryregister/discoveryregister.go | 2 +- pkg/common/discoveryregister/etcd/doc.go | 15 + pkg/common/discoveryregister/etcd/etcd.go | 289 ------------------ 5 files changed, 20 insertions(+), 294 deletions(-) create mode 100644 pkg/common/discoveryregister/etcd/doc.go delete mode 100644 pkg/common/discoveryregister/etcd/etcd.go diff --git a/go.mod b/go.mod index 2ec3e20b2..640b4a824 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.9 + github.com/openimsdk/tools v0.0.49-alpha.10 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -40,7 +40,6 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.18.2 github.com/stathat/consistent v1.0.0 - go.etcd.io/etcd/client/v3 v3.5.13 go.uber.org/automaxprocs v1.5.3 golang.org/x/sync v0.6.0 ) @@ -142,6 +141,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/etcd/api/v3 v3.5.13 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect + go.etcd.io/etcd/client/v3 v3.5.13 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect diff --git a/go.sum b/go.sum index edecead94..241bb96aa 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.9 h1:yoa3GS6t0d1mRv/S86niFBGDgSjy2EWWwBI5NAH1Kgk= -github.com/openimsdk/tools v0.0.49-alpha.9/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.10 h1:vcfMUGBClD3TsjsTd/Wb0R2WcpjCo8hupMqreb0dXFA= +github.com/openimsdk/tools v0.0.49-alpha.10/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 1085ec1ea..f37fc9c10 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -16,9 +16,9 @@ package discoveryregister import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" - getcd "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" + getcd "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/errs" "time" diff --git a/pkg/common/discoveryregister/etcd/doc.go b/pkg/common/discoveryregister/etcd/doc.go new file mode 100644 index 000000000..1da7508a1 --- /dev/null +++ b/pkg/common/discoveryregister/etcd/doc.go @@ -0,0 +1,15 @@ +// Copyright © 2024 OpenIM. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes // import "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go deleted file mode 100644 index dd8806378..000000000 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ /dev/null @@ -1,289 +0,0 @@ -package etcd - -import ( - "context" - "fmt" - "github.com/pkg/errors" - clientv3 "go.etcd.io/etcd/client/v3" - "go.etcd.io/etcd/client/v3/naming/endpoints" - "go.etcd.io/etcd/client/v3/naming/resolver" - "google.golang.org/grpc" - gresolver "google.golang.org/grpc/resolver" - "strings" - "sync" - "time" -) - -// ZkOption defines a function type for modifying clientv3.Config -type ZkOption func(*clientv3.Config) - -// SvcDiscoveryRegistryImpl implementation -type SvcDiscoveryRegistryImpl struct { - client *clientv3.Client - resolver gresolver.Builder - dialOptions []grpc.DialOption - serviceKey string - endpointMgr endpoints.Manager - leaseID clientv3.LeaseID - rpcRegisterTarget string - - rootDirectory string - - mu sync.RWMutex - connMap map[string][]*grpc.ClientConn -} - -// NewSvcDiscoveryRegistry creates a new service discovery registry implementation -func NewSvcDiscoveryRegistry(rootDirectory string, endpoints []string, options ...ZkOption) (*SvcDiscoveryRegistryImpl, error) { - cfg := clientv3.Config{ - Endpoints: endpoints, - DialTimeout: 5 * time.Second, - // Increase keep-alive queue capacity and message size - PermitWithoutStream: true, - MaxCallSendMsgSize: 10 * 1024 * 1024, // 10 MB - } - - // Apply provided options to the config - for _, opt := range options { - opt(&cfg) - } - - client, err := clientv3.New(cfg) - if err != nil { - return nil, err - } - r, err := resolver.NewBuilder(client) - if err != nil { - return nil, err - } - - s := &SvcDiscoveryRegistryImpl{ - client: client, - resolver: r, - rootDirectory: rootDirectory, - connMap: make(map[string][]*grpc.ClientConn), - } - go s.watchServiceChanges() - return s, nil -} - -// WithDialTimeout sets a custom dial timeout for the etcd client -func WithDialTimeout(timeout time.Duration) ZkOption { - return func(cfg *clientv3.Config) { - cfg.DialTimeout = timeout - } -} - -// WithMaxCallSendMsgSize sets a custom max call send message size for the etcd client -func WithMaxCallSendMsgSize(size int) ZkOption { - return func(cfg *clientv3.Config) { - cfg.MaxCallSendMsgSize = size - } -} - -// WithUsernameAndPassword sets a username and password for the etcd client -func WithUsernameAndPassword(username, password string) ZkOption { - return func(cfg *clientv3.Config) { - cfg.Username = username - cfg.Password = password - } -} - -// GetUserIdHashGatewayHost returns the gateway host for a given user ID hash -func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { - return "", nil -} - -// GetConns returns gRPC client connections for a given service name -func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - fullServiceKey := fmt.Sprintf("%s/%s", r.rootDirectory, serviceName) - r.mu.RLock() - defer r.mu.RUnlock() - return r.connMap[fullServiceKey], nil -} - -// GetConn returns a single gRPC client connection for a given service name -func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - target := fmt.Sprintf("etcd:///%s/%s", r.rootDirectory, serviceName) - return grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) -} - -// GetSelfConnTarget returns the connection target for the current service -func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { - return r.rpcRegisterTarget -} - -// AddOption appends gRPC dial options to the existing options -func (r *SvcDiscoveryRegistryImpl) AddOption(opts ...grpc.DialOption) { - r.dialOptions = append(r.dialOptions, opts...) -} - -// CloseConn closes a given gRPC client connection -func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { - conn.Close() -} - -// Register registers a new service endpoint with etcd -func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s/%s:%d", r.rootDirectory, serviceName, host, port) - em, err := endpoints.NewManager(r.client, r.rootDirectory+"/"+serviceName) - if err != nil { - return err - } - r.endpointMgr = em - - leaseResp, err := r.client.Grant(context.Background(), 30) // - if err != nil { - return err - } - r.leaseID = leaseResp.ID - - r.rpcRegisterTarget = fmt.Sprintf("%s:%d", host, port) - endpoint := endpoints.Endpoint{Addr: r.rpcRegisterTarget} - - err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) - if err != nil { - return err - } - - go r.keepAliveLease(r.leaseID) - return nil -} - -// keepAliveLease maintains the lease alive by sending keep-alive requests -func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { - ch, err := r.client.KeepAlive(context.Background(), leaseID) - if err != nil { - return - } - for ka := range ch { - if ka != nil { - } else { - return - } - } -} - -// watchServiceChanges watches for changes in the service directory -func (r *SvcDiscoveryRegistryImpl) watchServiceChanges() { - watchChan := r.client.Watch(context.Background(), r.rootDirectory, clientv3.WithPrefix()) - for watchResp := range watchChan { - updatedPrefixes := make(map[string]struct{}) // Create a set to track updated prefixes - - for _, event := range watchResp.Events { - prefix, _ := r.splitEndpoint(string(event.Kv.Key)) - if _, alreadyUpdated := updatedPrefixes[prefix]; !alreadyUpdated { - updatedPrefixes[prefix] = struct{}{} // Mark this prefix as updated - r.refreshConnMap(prefix) - } - } - } -} - -// refreshConnMap fetches the latest endpoints and updates the local map -func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { - r.mu.Lock() - defer r.mu.Unlock() - - fullPrefix := fmt.Sprintf("%s/", prefix) - resp, err := r.client.Get(context.Background(), fullPrefix, clientv3.WithPrefix()) - if err != nil { - return - } - r.connMap[prefix] = []*grpc.ClientConn{} // Update the connMap with new connections - for _, kv := range resp.Kvs { - _, addr := r.splitEndpoint(string(kv.Key)) - conn, err := grpc.DialContext(context.Background(), addr, append(r.dialOptions, grpc.WithResolvers(r.resolver))...) - if err != nil { - continue - } - r.connMap[prefix] = append(r.connMap[prefix], conn) - } -} - -// splitEndpoint splits the endpoint string into prefix and address -func (r *SvcDiscoveryRegistryImpl) splitEndpoint(input string) (string, string) { - lastSlashIndex := strings.LastIndex(input, "/") - if lastSlashIndex != -1 { - part1 := input[:lastSlashIndex] - part2 := input[lastSlashIndex+1:] - return part1, part2 - } - return input, "" -} - -// UnRegister removes the service endpoint from etcd -func (r *SvcDiscoveryRegistryImpl) UnRegister() error { - if r.endpointMgr == nil { - return fmt.Errorf("endpoint manager is not initialized") - } - err := r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) - if err != nil { - return err - } - return nil -} - -// Close closes the etcd client connection -func (r *SvcDiscoveryRegistryImpl) Close() { - if r.client != nil { - _ = r.client.Close() - } - - r.mu.Lock() - defer r.mu.Unlock() -} - -// Check verifies if etcd is running by checking the existence of the root node and optionally creates it with a lease -func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfNotExist bool, options ...ZkOption) error { - cfg := clientv3.Config{ - Endpoints: etcdServers, - } - for _, opt := range options { - opt(&cfg) - } - client, err := clientv3.New(cfg) - if err != nil { - return errors.Wrap(err, "failed to connect to etcd") - } - defer client.Close() - - var opCtx context.Context - var cancel context.CancelFunc - if cfg.DialTimeout != 0 { - opCtx, cancel = context.WithTimeout(ctx, cfg.DialTimeout) - } else { - opCtx, cancel = context.WithTimeout(ctx, 10*time.Second) - } - defer cancel() - - resp, err := client.Get(opCtx, etcdRoot) - if err != nil { - return errors.Wrap(err, "failed to get the root node from etcd") - } - - if len(resp.Kvs) == 0 { - if createIfNotExist { - var leaseTTL int64 = 10 - var leaseResp *clientv3.LeaseGrantResponse - if leaseTTL > 0 { - leaseResp, err = client.Grant(opCtx, leaseTTL) - if err != nil { - return errors.Wrap(err, "failed to create lease in etcd") - } - } - putOpts := []clientv3.OpOption{} - if leaseResp != nil { - putOpts = append(putOpts, clientv3.WithLease(leaseResp.ID)) - } - - _, err := client.Put(opCtx, etcdRoot, "", putOpts...) - if err != nil { - return errors.Wrap(err, "failed to create the root node in etcd") - } - } else { - return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) - } - } - return nil -} From 6b082611cef1ec8c1b4ddcc3cacaff3952e43c46 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 10:52:08 +0800 Subject: [PATCH 057/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/discoveryregister.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index f37fc9c10..1debc54e1 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -18,7 +18,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" - getcd "github.com/openimsdk/tools/discovery/etcd" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/errs" "time" @@ -39,12 +39,12 @@ func NewDiscoveryRegister(discovery *config.Discovery, share *config.Share) (dis case "k8s": return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) case "etcd": - return getcd.NewSvcDiscoveryRegistry( + return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, discovery.Etcd.Address, - getcd.WithDialTimeout(10*time.Second), - getcd.WithMaxCallSendMsgSize(20*1024*1024), - getcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) + etcd.WithDialTimeout(10*time.Second), + etcd.WithMaxCallSendMsgSize(20*1024*1024), + etcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) case "direct": //return direct.NewConnDirect(config) default: From 850c8253f059e39b04988b8efee4fe262f56948b Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 10:53:06 +0800 Subject: [PATCH 058/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/discoveryregister.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 1debc54e1..559c937c1 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -45,10 +45,7 @@ func NewDiscoveryRegister(discovery *config.Discovery, share *config.Share) (dis etcd.WithDialTimeout(10*time.Second), etcd.WithMaxCallSendMsgSize(20*1024*1024), etcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) - case "direct": - //return direct.NewConnDirect(config) default: return nil, errs.New("unsupported discovery type", "type", discovery.Enable).Wrap() } - return nil, nil } From c190e30a16b420788c944e405e9148fbb11eca3d Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:06:44 +0800 Subject: [PATCH 059/143] Add etcd as a service discovery mechanism --- tools/check-component/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 5d4435e26..51daa0782 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -20,9 +20,9 @@ import ( "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/minio" From 22a07a88f1ec6a9d28a328c4edb194c9457b6bc5 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:13:29 +0800 Subject: [PATCH 060/143] Add etcd as a service discovery mechanism --- tools/check-component/main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 51daa0782..5fa84ac36 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -139,28 +139,28 @@ func main() { func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig *config.Redis, kafkaConfig *config.Kafka, minioConfig *config.Minio, discovery *config.Discovery, maxRetry int) error { checksDone := make(map[string]bool) - checks := map[string]func() error{ - "Mongo": func() error { + checks := map[string]func(ctx context.Context) error{ + "Mongo": func(ctx context.Context) error { return CheckMongo(ctx, mongoConfig) }, - "Redis": func() error { + "Redis": func(ctx context.Context) error { return CheckRedis(ctx, redisConfig) }, - "Kafka": func() error { + "Kafka": func(ctx context.Context) error { return CheckKafka(ctx, kafkaConfig) }, } if minioConfig != nil { - checks["MinIO"] = func() error { + checks["MinIO"] = func(ctx context.Context) error { return CheckMinIO(ctx, minioConfig) } } if discovery.Enable == "etcd" { - checks["Etcd"] = func() error { + checks["Etcd"] = func(ctx context.Context) error { return CheckEtcd(ctx, &discovery.Etcd) } } else if discovery.Enable == "zookeeper" { - checks["Zookeeper"] = func() error { + checks["Zookeeper"] = func(ctx context.Context) error { return CheckZookeeper(ctx, &discovery.ZooKeeper) } } @@ -169,7 +169,7 @@ func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig * allSuccess := true for name, check := range checks { if !checksDone[name] { - if err := check(); err != nil { + if err := check(ctx); err != nil { fmt.Printf("%s check failed: %v\n", name, err) allSuccess = false } else { From adb716424ccb8b63fed321f4efd0d0857031ce33 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:35:52 +0800 Subject: [PATCH 061/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 9c9189d37..887f4208f 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -61,7 +61,12 @@ func NewDefaultAllNode(disCov discovery.SvcDiscoveryRegistry, config *Config) *D func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) { conns, err := d.disCov.GetConns(ctx, d.config.Share.RpcRegisterName.MessageGateway) - log.ZDebug(ctx, "get gateway conn", "conn length", len(conns)) + if len(conns) == 0 { + log.ZWarn(ctx, "get gateway conn 0 ", nil) + } else { + log.ZDebug(ctx, "get gateway conn", "conn length", len(conns)) + } + if err != nil { return nil, err } From 902338a894002f03fdd33c7873cae4c6a847db26 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:47:52 +0800 Subject: [PATCH 062/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 640b4a824..7d20cc62b 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.10 + github.com/openimsdk/tools v0.0.49-alpha.11 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 241bb96aa..d010c6736 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.10 h1:vcfMUGBClD3TsjsTd/Wb0R2WcpjCo8hupMqreb0dXFA= -github.com/openimsdk/tools v0.0.49-alpha.10/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.11 h1:LybY7RVMwTqnBA5dLuZtI0SyBWyZj2oLqyzIieoNo5E= +github.com/openimsdk/tools v0.0.49-alpha.11/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 33ad667e7bb6f3a5d8ff6bb72ce9688ae803a290 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:58:37 +0800 Subject: [PATCH 063/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7d20cc62b..841e014c3 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.11 + github.com/openimsdk/tools v0.0.49-alpha.12 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 From 1c623d068d72fe028569a0de51db7702d8bca06d Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:59:10 +0800 Subject: [PATCH 064/143] Add etcd as a service discovery mechanism --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index d010c6736..d94d4c2f5 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.11 h1:LybY7RVMwTqnBA5dLuZtI0SyBWyZj2oLqyzIieoNo5E= -github.com/openimsdk/tools v0.0.49-alpha.11/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.12 h1:vsr63W1kHW1dEw9yelMhmr72WmsrjKfs2vXww3upfWI= +github.com/openimsdk/tools v0.0.49-alpha.12/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From eaa2054b102362473ddb9124c85599d329f505a0 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 12:06:03 +0800 Subject: [PATCH 065/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 841e014c3..73ec1e59a 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.12 + github.com/openimsdk/tools v0.0.49-alpha.13 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index d94d4c2f5..4c331bc91 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.12 h1:vsr63W1kHW1dEw9yelMhmr72WmsrjKfs2vXww3upfWI= -github.com/openimsdk/tools v0.0.49-alpha.12/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.13 h1:u6r9eT90Co1gz/ukAjFAtxykU3JP4REWCgAy7WijVZ8= +github.com/openimsdk/tools v0.0.49-alpha.13/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 544bb96ef990dbbd32581cf77385671ca5364168 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 12:15:22 +0800 Subject: [PATCH 066/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 887f4208f..55b3ac481 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -91,6 +91,7 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M msgClient := msggateway.NewMsgGatewayClient(conn) reply, err := msgClient.SuperGroupOnlineBatchPushOneMsg(ctx, input) if err != nil { + log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ", err, input.String()) return nil } From facdb10155ce04d9b0cbd045234a0a8fadae41f4 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 12:18:09 +0800 Subject: [PATCH 067/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 55b3ac481..9a2c52f01 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -91,7 +91,7 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M msgClient := msggateway.NewMsgGatewayClient(conn) reply, err := msgClient.SuperGroupOnlineBatchPushOneMsg(ctx, input) if err != nil { - log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ", err, input.String()) + log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ", err, "req:", input.String()) return nil } From 267733cff99a6ac03ff554337f862c8f4dd21ec5 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 12:26:50 +0800 Subject: [PATCH 068/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 9a2c52f01..a61399fb6 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -87,6 +87,7 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M // Online push message for _, conn := range conns { conn := conn // loop var safe + ctx := ctx wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(conn) reply, err := msgClient.SuperGroupOnlineBatchPushOneMsg(ctx, input) From 82ed74852eacbfae5f516cdcde37de856722df05 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 14:41:32 +0800 Subject: [PATCH 069/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 73ec1e59a..bc86e2efd 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.13 + github.com/openimsdk/tools v0.0.49-alpha.14 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 4c331bc91..8f2828e09 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.13 h1:u6r9eT90Co1gz/ukAjFAtxykU3JP4REWCgAy7WijVZ8= -github.com/openimsdk/tools v0.0.49-alpha.13/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.14 h1:1WigfCZhLQRYDdBjZRhf2ksFM4CWmMWd1y6RC36T02A= +github.com/openimsdk/tools v0.0.49-alpha.14/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 23b40867ce6b1627450fb12dab75ab29a16b3225 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 14:49:14 +0800 Subject: [PATCH 070/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bc86e2efd..22efc0e1b 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.14 + github.com/openimsdk/tools v0.0.49-alpha.15 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 From daf5db654bbee88f2588429798d7b3dee4336222 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 14:50:30 +0800 Subject: [PATCH 071/143] Add etcd as a service discovery mechanism --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 8f2828e09..a1e4832cd 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.14 h1:1WigfCZhLQRYDdBjZRhf2ksFM4CWmMWd1y6RC36T02A= -github.com/openimsdk/tools v0.0.49-alpha.14/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.15 h1:m9bvSLBnqLZ60pkeB2lvYOsprLnLphaFetoZCS75q0E= +github.com/openimsdk/tools v0.0.49-alpha.15/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From ff5318d92ae0b3208f25020fe552f0dfec3d5e10 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 15:23:10 +0800 Subject: [PATCH 072/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 22efc0e1b..9f48b93b0 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.15 + github.com/openimsdk/tools v0.0.49-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index a1e4832cd..acb199acd 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.15 h1:m9bvSLBnqLZ60pkeB2lvYOsprLnLphaFetoZCS75q0E= -github.com/openimsdk/tools v0.0.49-alpha.15/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.16 h1:AYL4DsE/k1uyfT5kcsdPbbloU5dSIGVERGstL0OgLwg= +github.com/openimsdk/tools v0.0.49-alpha.16/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From e349e4427b55119aff309d7ed946bd95a6b6383e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 15:30:10 +0800 Subject: [PATCH 073/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9f48b93b0..4a2d7a002 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.16 + github.com/openimsdk/tools v0.0.49-alpha.17 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index acb199acd..801783289 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.16 h1:AYL4DsE/k1uyfT5kcsdPbbloU5dSIGVERGstL0OgLwg= -github.com/openimsdk/tools v0.0.49-alpha.16/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.17 h1:cqPwj7AhVraRjAwLBk4BFKp9ASCbnh2GFFSRe9QgYCk= +github.com/openimsdk/tools v0.0.49-alpha.17/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 797b13e4a4722d04c413a2fc557b3d24437d4269 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 17:00:20 +0800 Subject: [PATCH 074/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4a2d7a002..e9777eaa8 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.17 + github.com/openimsdk/tools v0.0.49-alpha.18 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 801783289..3acb4709c 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.17 h1:cqPwj7AhVraRjAwLBk4BFKp9ASCbnh2GFFSRe9QgYCk= -github.com/openimsdk/tools v0.0.49-alpha.17/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.18 h1:ARQeCiRmExvtB6XYItegThuV63JGOTxddwhSLHYXd78= +github.com/openimsdk/tools v0.0.49-alpha.18/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 62b4a6a0ca44b5d1e3e35210bbbc8115b1adb3a3 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:30:12 +0800 Subject: [PATCH 075/143] add etcd --- go.mod | 4 +- go.sum | 4 +- .../discoveryregister/discoveryregister.go | 38 +++--- pkg/common/discoveryregister/etcd/etcd.go | 110 ++++++++++++++++++ 4 files changed, 136 insertions(+), 20 deletions(-) create mode 100644 pkg/common/discoveryregister/etcd/etcd.go diff --git a/go.mod b/go.mod index e9777eaa8..6b6de7804 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.18 + github.com/openimsdk/tools v0.0.49-alpha.6 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -40,6 +40,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.18.2 github.com/stathat/consistent v1.0.0 + go.etcd.io/etcd/client/v3 v3.5.13 go.uber.org/automaxprocs v1.5.3 golang.org/x/sync v0.6.0 ) @@ -141,7 +142,6 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/etcd/api/v3 v3.5.13 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect - go.etcd.io/etcd/client/v3 v3.5.13 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect diff --git a/go.sum b/go.sum index 3acb4709c..9a1bd14aa 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.18 h1:ARQeCiRmExvtB6XYItegThuV63JGOTxddwhSLHYXd78= -github.com/openimsdk/tools v0.0.49-alpha.18/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.6 h1:I3HSyNWqOMTzZ1DlD0ZDKAgVzxGAFnN9EYW0xkA+8To= +github.com/openimsdk/tools v0.0.49-alpha.6/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 559c937c1..4aeee67bc 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -16,36 +16,42 @@ package discoveryregister import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" + getcd "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" - "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/errs" "time" ) +const ( + zookeeperConst = "zookeeper" + kubenetesConst = "k8s" + directConst = "direct" + etcd = "etcd" +) + // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(discovery *config.Discovery, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { - switch discovery.Enable { - case "zookeeper": +func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry1, error) { + switch share.Env { + case zookeeperConst: + return zookeeper.NewZkClient( - discovery.ZooKeeper.Address, - discovery.ZooKeeper.Schema, + zookeeperConfig.Address, + zookeeperConfig.Schema, zookeeper.WithFreq(time.Hour), - zookeeper.WithUserNameAndPassword(discovery.ZooKeeper.Username, discovery.ZooKeeper.Password), + zookeeper.WithUserNameAndPassword(zookeeperConfig.Username, zookeeperConfig.Password), zookeeper.WithRoundRobin(), zookeeper.WithTimeout(10), ) - case "k8s": + case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) - case "etcd": - return etcd.NewSvcDiscoveryRegistry( - discovery.Etcd.RootDirectory, - discovery.Etcd.Address, - etcd.WithDialTimeout(10*time.Second), - etcd.WithMaxCallSendMsgSize(20*1024*1024), - etcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) + case etcd: + return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) + case directConst: + //return direct.NewConnDirect(config) default: - return nil, errs.New("unsupported discovery type", "type", discovery.Enable).Wrap() + return nil, errs.New("unsupported discovery type", "type", share.Env).Wrap() } + return nil, nil } diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go new file mode 100644 index 000000000..5c6bae849 --- /dev/null +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -0,0 +1,110 @@ +package etcd + +import ( + "context" + "fmt" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/naming/endpoints" + "go.etcd.io/etcd/client/v3/naming/resolver" + "google.golang.org/grpc" + gresolver "google.golang.org/grpc/resolver" + + "log" + "time" +) + +// SvcDiscoveryRegistryImpl implementation +type SvcDiscoveryRegistryImpl struct { + client *clientv3.Client + resolver gresolver.Builder + dialOptions []grpc.DialOption + serviceKey string + endpointMgr endpoints.Manager + leaseID clientv3.LeaseID + schema string +} + +func NewSvcDiscoveryRegistry(schema string, endpoints []string) (*SvcDiscoveryRegistryImpl, error) { + cfg := clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 5 * time.Second, + } + client, err := clientv3.New(cfg) + if err != nil { + return nil, err + } + r, err := resolver.NewBuilder(client) + if err != nil { + return nil, err + } + return &SvcDiscoveryRegistryImpl{ + client: client, + resolver: r, + schema: schema, + }, nil +} + +func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { + target := fmt.Sprintf("%s:///%s", r.schema, serviceName) + conn, err := grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) + if err != nil { + return nil, err + } + return []*grpc.ClientConn{conn}, nil +} + +func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + target := fmt.Sprintf("%s:///%s", r.schema, serviceName) + return grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) +} + +func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { + return fmt.Sprintf("%s:///%s", r.schema, r.serviceKey) +} + +func (r *SvcDiscoveryRegistryImpl) AddOption(opts ...grpc.DialOption) { + r.dialOptions = append(r.dialOptions, opts...) +} + +func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { + if err := conn.Close(); err != nil { + log.Printf("Failed to close connection: %v", err) + } +} + +func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { + r.serviceKey = fmt.Sprintf("%s/%s:%d", serviceName, host, port) + em, err := endpoints.NewManager(r.client, serviceName) + if err != nil { + return err + } + r.endpointMgr = em + + leaseResp, err := r.client.Grant(context.Background(), 5) + if err != nil { + return err + } + r.leaseID = leaseResp.ID + + endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} + err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) + if err != nil { + return err + } + + _, err = r.client.KeepAlive(context.Background(), r.leaseID) + return err +} + +func (r *SvcDiscoveryRegistryImpl) UnRegister() error { + if r.endpointMgr == nil { + return fmt.Errorf("endpoint manager is not initialized") + } + return r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) +} + +func (r *SvcDiscoveryRegistryImpl) Close() { + if r.client != nil { + _ = r.client.Close() + } +} From 1c8d2bbacd4e112fbd1b2b7802518b3f6bda5772 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:37:43 +0800 Subject: [PATCH 076/143] add etcd --- pkg/common/startrpc/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index a36bcfe1c..01cc1c518 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -46,7 +46,7 @@ import ( // Start rpc server. func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusConfig *config2.Prometheus, listenIP, registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { + config T, client discovery.SvcDiscoveryRegistry1, server *grpc.Server) error, options ...grpc.ServerOption) error { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) if err != nil { From 6bc6cf898f7084dc39702a0bbe4121e73bbe3f3e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:54:16 +0800 Subject: [PATCH 077/143] add etcd --- pkg/common/discoveryregister/discoveryregister.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 4aeee67bc..da1822f36 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -28,7 +28,7 @@ const ( zookeeperConst = "zookeeper" kubenetesConst = "k8s" directConst = "direct" - etcd = "etcd" + etcdConst = "etcd" ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. @@ -46,7 +46,7 @@ func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share ) case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) - case etcd: + case etcdConst: return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) case directConst: //return direct.NewConnDirect(config) From 1b3bc6a6f6ead6ae52f7036bb0cd819264605a1f Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Fri, 10 May 2024 21:55:54 +0800 Subject: [PATCH 078/143] add etcd --- pkg/common/discoveryregister/discoveryregister.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index da1822f36..0868e6f51 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -33,6 +33,7 @@ const ( // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry1, error) { + return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) switch share.Env { case zookeeperConst: From 213db4c0661f78b22fb241d756ac09e4d4788bd6 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 09:25:08 +0800 Subject: [PATCH 079/143] add etcd --- go.mod | 2 +- go.sum | 4 ++-- pkg/common/discoveryregister/discoveryregister.go | 4 +--- pkg/common/startrpc/start.go | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 6b6de7804..467b58675 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.6 + github.com/openimsdk/tools v0.0.49-alpha.8 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 9a1bd14aa..fa1b15aa3 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.6 h1:I3HSyNWqOMTzZ1DlD0ZDKAgVzxGAFnN9EYW0xkA+8To= -github.com/openimsdk/tools v0.0.49-alpha.6/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.8 h1:/l7u13eT6PByF21JOYuIteEUa9/XR63VwJWg0EHx3Ew= +github.com/openimsdk/tools v0.0.49-alpha.8/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 0868e6f51..31bee9267 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -32,11 +32,9 @@ const ( ) // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry1, error) { - return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) +func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { switch share.Env { case zookeeperConst: - return zookeeper.NewZkClient( zookeeperConfig.Address, zookeeperConfig.Schema, diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 01cc1c518..a36bcfe1c 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -46,7 +46,7 @@ import ( // Start rpc server. func Start[T any](ctx context.Context, discovery *config2.Discovery, prometheusConfig *config2.Prometheus, listenIP, registerIP string, rpcPorts []int, index int, rpcRegisterName string, share *config2.Share, config T, rpcFn func(ctx context.Context, - config T, client discovery.SvcDiscoveryRegistry1, server *grpc.Server) error, options ...grpc.ServerOption) error { + config T, client discovery.SvcDiscoveryRegistry, server *grpc.Server) error, options ...grpc.ServerOption) error { rpcPort, err := datautil.GetElemByIndex(rpcPorts, index) if err != nil { From c30625a1f00ea14a614491b7fcdd530ec223ba59 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 09:31:16 +0800 Subject: [PATCH 080/143] add etcd --- go.mod | 2 +- go.sum | 4 ++-- pkg/common/discoveryregister/etcd/etcd.go | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 467b58675..2ec3e20b2 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.8 + github.com/openimsdk/tools v0.0.49-alpha.9 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index fa1b15aa3..edecead94 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.8 h1:/l7u13eT6PByF21JOYuIteEUa9/XR63VwJWg0EHx3Ew= -github.com/openimsdk/tools v0.0.49-alpha.8/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.9 h1:yoa3GS6t0d1mRv/S86niFBGDgSjy2EWWwBI5NAH1Kgk= +github.com/openimsdk/tools v0.0.49-alpha.9/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 5c6bae849..ce76fae15 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -44,6 +44,9 @@ func NewSvcDiscoveryRegistry(schema string, endpoints []string) (*SvcDiscoveryRe }, nil } +func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { + return "", nil +} func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) conn, err := grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) From c28579f6d0f05814e5283fd32b84ea7d8380f358 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 09:50:11 +0800 Subject: [PATCH 081/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index ce76fae15..db27ebf8f 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -83,7 +83,7 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, } r.endpointMgr = em - leaseResp, err := r.client.Grant(context.Background(), 5) + leaseResp, err := r.client.Grant(context.Background(), 30) if err != nil { return err } @@ -91,11 +91,6 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) - if err != nil { - return err - } - - _, err = r.client.KeepAlive(context.Background(), r.leaseID) return err } From 656905efa2cc912b7a7faa7ec62927b9c1558e03 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 09:56:37 +0800 Subject: [PATCH 082/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index db27ebf8f..cb8284e72 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -76,7 +76,7 @@ func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { } func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s:%d", serviceName, host, port) + r.serviceKey = fmt.Sprintf("%s/%s/%d", serviceName, host, port) em, err := endpoints.NewManager(r.client, serviceName) if err != nil { return err From babc2874a4ef64c7bbda06e00424e68026f637a0 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:09:40 +0800 Subject: [PATCH 083/143] add etcd --- pkg/common/discoveryregister/discoveryregister.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 31bee9267..c2deb6cf2 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -46,7 +46,7 @@ func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) case etcdConst: - return getcd.NewSvcDiscoveryRegistry("openim", []string{"http://localhost:2379"}) + return getcd.NewSvcDiscoveryRegistry("etcd", []string{"http://localhost:2379"}) case directConst: //return direct.NewConnDirect(config) default: From 1b232a1370961672e92c148bb0a1d32fe3c564f7 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:19:05 +0800 Subject: [PATCH 084/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index cb8284e72..81ec77600 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -49,7 +49,7 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, } func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) - conn, err := grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) + conn, err := grpc.DialContext(ctx, target, grpc.WithResolvers(r.resolver)) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) - return grpc.DialContext(ctx, target, append(r.dialOptions, opts...)...) + return grpc.DialContext(ctx, target, grpc.WithResolvers(r.resolver)) } func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { From 8dc6c490d9e504e04e4c9b30627b962ad9c24c2c Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:30:47 +0800 Subject: [PATCH 085/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 81ec77600..871bc07a0 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -49,7 +49,7 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, } func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) - conn, err := grpc.DialContext(ctx, target, grpc.WithResolvers(r.resolver)) + conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { target := fmt.Sprintf("%s:///%s", r.schema, serviceName) - return grpc.DialContext(ctx, target, grpc.WithResolvers(r.resolver)) + return grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) } func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { From 140d9625fe03fd0e1b22f0ec67c2d626cea8ee95 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:55:57 +0800 Subject: [PATCH 086/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 871bc07a0..9edb48d82 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -76,7 +76,7 @@ func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { } func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s/%d", serviceName, host, port) + r.serviceKey = fmt.Sprintf("%s/%s-%d", serviceName, host, port) em, err := endpoints.NewManager(r.client, serviceName) if err != nil { return err @@ -92,6 +92,16 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) return err + + lease, _ := r.client.Grant(context.TODO(), 30) + + em, err = endpoints.NewManager(r.client, "foo/bar/my-service") + if err != nil { + return err + } + + err := em.AddEndpoint(context.TODO(), "foo/bar/my-service/e1", endpoints.Endpoint{Addr: "1.2.3.4"}, clientv3.WithLease(lease.ID)) + } func (r *SvcDiscoveryRegistryImpl) UnRegister() error { From ed3b1e13d8f150c0310b26767a176895a19fb0e1 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 10:59:29 +0800 Subject: [PATCH 087/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 9edb48d82..04bd57ca5 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -92,15 +92,15 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) return err - - lease, _ := r.client.Grant(context.TODO(), 30) - - em, err = endpoints.NewManager(r.client, "foo/bar/my-service") - if err != nil { - return err - } - - err := em.AddEndpoint(context.TODO(), "foo/bar/my-service/e1", endpoints.Endpoint{Addr: "1.2.3.4"}, clientv3.WithLease(lease.ID)) + // + //lease, _ := r.client.Grant(context.TODO(), 30) + // + //em, err = endpoints.NewManager(r.client, "foo/bar/my-service") + //if err != nil { + // return err + //} + // + //err := em.AddEndpoint(context.TODO(), "foo/bar/my-service/e1", endpoints.Endpoint{Addr: "1.2.3.4"}, clientv3.WithLease(lease.ID)) } From a08d0d7218f59c57d838296be62e55f7bf4ebd7c Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 11:22:49 +0800 Subject: [PATCH 088/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 04bd57ca5..e627dfedb 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -91,17 +91,12 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) - return err - // - //lease, _ := r.client.Grant(context.TODO(), 30) - // - //em, err = endpoints.NewManager(r.client, "foo/bar/my-service") - //if err != nil { - // return err - //} - // - //err := em.AddEndpoint(context.TODO(), "foo/bar/my-service/e1", endpoints.Endpoint{Addr: "1.2.3.4"}, clientv3.WithLease(lease.ID)) + if err != nil { + return err + } + _, kaErr := r.client.KeepAlive(context.Background(), r.leaseID) + return kaErr } func (r *SvcDiscoveryRegistryImpl) UnRegister() error { From 02cc4a209790baf7ef7c0e0a0e683cfbad898f80 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 11:40:44 +0800 Subject: [PATCH 089/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index e627dfedb..cc94f259a 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -95,8 +95,26 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, return err } - _, kaErr := r.client.KeepAlive(context.Background(), r.leaseID) - return kaErr + go r.keepAliveLease(r.leaseID) + + return nil +} + +func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { + ch, err := r.client.KeepAlive(context.Background(), leaseID) + if err != nil { + log.Printf("Failed to keep lease alive: %v", err) + return + } + + for ka := range ch { + if ka != nil { + fmt.Printf("Received lease keep-alive response: %v", ka) + } else { + fmt.Printf("Lease keep-alive response channel closed") + break + } + } } func (r *SvcDiscoveryRegistryImpl) UnRegister() error { From 39b32fa6990966fcef3748ba597cff55e7fd052e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 11:44:12 +0800 Subject: [PATCH 090/143] add etcd --- pkg/common/discoveryregister/etcd/etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index cc94f259a..62d6df150 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -109,7 +109,7 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { for ka := range ch { if ka != nil { - fmt.Printf("Received lease keep-alive response: %v", ka) + fmt.Printf("Received lease keep-alive response: %v\n", ka) } else { fmt.Printf("Lease keep-alive response channel closed") break From dfa20538f43ac5e978acbe9df67cf46bdfe6cd23 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 12:03:29 +0800 Subject: [PATCH 091/143] add etcd --- config/discovery.yml | 7 +++++-- pkg/common/discoveryregister/discoveryregister.go | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 3d96ff9b6..6e0922ede 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,13 +1,16 @@ + enable: "etcd" etcd: - rootDirectory: openim - address: [ localhost:12379 ] + schema: openim + address: [ http://localhost:2379 ] username: '' password: '' + zookeeper: schema: openim address: [ localhost:12181 ] username: '' password: '' + diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index c2deb6cf2..dbf16eda0 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -46,7 +46,7 @@ func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share case kubenetesConst: return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) case etcdConst: - return getcd.NewSvcDiscoveryRegistry("etcd", []string{"http://localhost:2379"}) + return getcd.NewSvcDiscoveryRegistry("etcd", []string{"localhost:2379"}) case directConst: //return direct.NewConnDirect(config) default: From a4d6b2a3b9767f60cd6254016daa75a2d8846265 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 12:18:18 +0800 Subject: [PATCH 092/143] add etcd --- config/discovery.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 6e0922ede..2cc54be72 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,12 +1,12 @@ enable: "etcd" etcd: - schema: openim - address: [ http://localhost:2379 ] + address: [ localhost:12379 ] username: '' password: '' + zookeeper: schema: openim address: [ localhost:12181 ] From db906fcea0272abb24aa542a89ecc2fe7188186d Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 15:40:17 +0800 Subject: [PATCH 093/143] Add etcd as a service discovery mechanism --- config/discovery.yml | 5 +- internal/msgtransfer/init.go | 15 ++-- internal/rpc/conversation/conversaion.go | 7 +- .../discoveryregister/discoveryregister.go | 34 ++++--- pkg/common/discoveryregister/etcd/etcd.go | 90 ++++++++++++++----- 5 files changed, 96 insertions(+), 55 deletions(-) diff --git a/config/discovery.yml b/config/discovery.yml index 2cc54be72..3d96ff9b6 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -1,16 +1,13 @@ - enable: "etcd" etcd: + rootDirectory: openim address: [ localhost:12379 ] username: '' password: '' - - zookeeper: schema: openim address: [ localhost:12181 ] username: '' password: '' - diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 3384b8493..8f72e979d 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -56,13 +56,14 @@ type MsgTransfer struct { } type Config struct { - MsgTransfer config.MsgTransfer - RedisConfig config.Redis - MongodbConfig config.Mongo - KafkaConfig config.Kafka - Share config.Share - WebhooksConfig config.Webhooks - Discovery config.Discovery + MsgTransfer config.MsgTransfer + RedisConfig config.Redis + MongodbConfig config.Mongo + KafkaConfig config.Kafka + ZookeeperConfig config.ZooKeeper + Share config.Share + WebhooksConfig config.Webhooks + Discovery config.Discovery } func Start(ctx context.Context, index int, config *Config) error { diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 4c7828610..ec7522212 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -48,9 +48,10 @@ type conversationServer struct { } type Config struct { - RpcConfig config.Conversation - RedisConfig config.Redis - MongodbConfig config.Mongo + RpcConfig config.Conversation + RedisConfig config.Redis + MongodbConfig config.Mongo + // ZookeeperConfig config.ZooKeeper NotificationConfig config.Notification Share config.Share LocalCacheConfig config.LocalCache diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index dbf16eda0..1085ec1ea 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -24,33 +24,31 @@ import ( "time" ) -const ( - zookeeperConst = "zookeeper" - kubenetesConst = "k8s" - directConst = "direct" - etcdConst = "etcd" -) - // NewDiscoveryRegister creates a new service discovery and registry client based on the provided environment type. -func NewDiscoveryRegister(zookeeperConfig *config.ZooKeeper, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { - switch share.Env { - case zookeeperConst: +func NewDiscoveryRegister(discovery *config.Discovery, share *config.Share) (discovery.SvcDiscoveryRegistry, error) { + switch discovery.Enable { + case "zookeeper": return zookeeper.NewZkClient( - zookeeperConfig.Address, - zookeeperConfig.Schema, + discovery.ZooKeeper.Address, + discovery.ZooKeeper.Schema, zookeeper.WithFreq(time.Hour), - zookeeper.WithUserNameAndPassword(zookeeperConfig.Username, zookeeperConfig.Password), + zookeeper.WithUserNameAndPassword(discovery.ZooKeeper.Username, discovery.ZooKeeper.Password), zookeeper.WithRoundRobin(), zookeeper.WithTimeout(10), ) - case kubenetesConst: + case "k8s": return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) - case etcdConst: - return getcd.NewSvcDiscoveryRegistry("etcd", []string{"localhost:2379"}) - case directConst: + case "etcd": + return getcd.NewSvcDiscoveryRegistry( + discovery.Etcd.RootDirectory, + discovery.Etcd.Address, + getcd.WithDialTimeout(10*time.Second), + getcd.WithMaxCallSendMsgSize(20*1024*1024), + getcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) + case "direct": //return direct.NewConnDirect(config) default: - return nil, errs.New("unsupported discovery type", "type", share.Env).Wrap() + return nil, errs.New("unsupported discovery type", "type", discovery.Enable).Wrap() } return nil, nil } diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 62d6df150..904f83dd0 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -8,27 +8,38 @@ import ( "go.etcd.io/etcd/client/v3/naming/resolver" "google.golang.org/grpc" gresolver "google.golang.org/grpc/resolver" - - "log" "time" ) +// ZkOption defines a function type for modifying clientv3.Config +type ZkOption func(*clientv3.Config) + // SvcDiscoveryRegistryImpl implementation type SvcDiscoveryRegistryImpl struct { - client *clientv3.Client - resolver gresolver.Builder - dialOptions []grpc.DialOption - serviceKey string - endpointMgr endpoints.Manager - leaseID clientv3.LeaseID - schema string + client *clientv3.Client + resolver gresolver.Builder + dialOptions []grpc.DialOption + serviceKey string + endpointMgr endpoints.Manager + leaseID clientv3.LeaseID + rootDirectory string } -func NewSvcDiscoveryRegistry(schema string, endpoints []string) (*SvcDiscoveryRegistryImpl, error) { +// NewSvcDiscoveryRegistry creates a new service discovery registry implementation +func NewSvcDiscoveryRegistry(rootDirectory string, endpoints []string, options ...ZkOption) (*SvcDiscoveryRegistryImpl, error) { cfg := clientv3.Config{ Endpoints: endpoints, DialTimeout: 5 * time.Second, + // Increase keep-alive queue capacity and message size + PermitWithoutStream: true, + MaxCallSendMsgSize: 10 * 1024 * 1024, // 10 MB } + + // Apply provided options to the config + for _, opt := range options { + opt(&cfg) + } + client, err := clientv3.New(cfg) if err != nil { return nil, err @@ -38,17 +49,42 @@ func NewSvcDiscoveryRegistry(schema string, endpoints []string) (*SvcDiscoveryRe return nil, err } return &SvcDiscoveryRegistryImpl{ - client: client, - resolver: r, - schema: schema, + client: client, + resolver: r, + rootDirectory: rootDirectory, }, nil } +// WithDialTimeout sets a custom dial timeout for the etcd client +func WithDialTimeout(timeout time.Duration) ZkOption { + return func(cfg *clientv3.Config) { + cfg.DialTimeout = timeout + } +} + +// WithMaxCallSendMsgSize sets a custom max call send message size for the etcd client +func WithMaxCallSendMsgSize(size int) ZkOption { + return func(cfg *clientv3.Config) { + cfg.MaxCallSendMsgSize = size + } +} + +// WithUsernameAndPassword sets a username and password for the etcd client +func WithUsernameAndPassword(username, password string) ZkOption { + return func(cfg *clientv3.Config) { + cfg.Username = username + cfg.Password = password + } +} + +// GetUserIdHashGatewayHost returns the gateway host for a given user ID hash func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { return "", nil } + +// GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - target := fmt.Sprintf("%s:///%s", r.schema, serviceName) + target := fmt.Sprintf("etcd:///%s", serviceName) conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { return nil, err @@ -56,34 +92,39 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str return []*grpc.ClientConn{conn}, nil } +// GetConn returns a single gRPC client connection for a given service name func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - target := fmt.Sprintf("%s:///%s", r.schema, serviceName) + target := fmt.Sprintf("etcd:///%s", serviceName) return grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) } +// GetSelfConnTarget returns the connection target for the current service func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { - return fmt.Sprintf("%s:///%s", r.schema, r.serviceKey) + return fmt.Sprintf("etcd:///%s", r.serviceKey) } +// AddOption appends gRPC dial options to the existing options func (r *SvcDiscoveryRegistryImpl) AddOption(opts ...grpc.DialOption) { r.dialOptions = append(r.dialOptions, opts...) } +// CloseConn closes a given gRPC client connection func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { if err := conn.Close(); err != nil { - log.Printf("Failed to close connection: %v", err) + fmt.Printf("Failed to close connection: %v\n", err) } } +// Register registers a new service endpoint with etcd func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s-%d", serviceName, host, port) - em, err := endpoints.NewManager(r.client, serviceName) + r.serviceKey = fmt.Sprintf("%s/%s/%s-%d", r.rootDirectory, serviceName, host, port) + em, err := endpoints.NewManager(r.client, r.rootDirectory+"/"+serviceName) if err != nil { return err } r.endpointMgr = em - leaseResp, err := r.client.Grant(context.Background(), 30) + leaseResp, err := r.client.Grant(context.Background(), 60) // Increase TTL time if err != nil { return err } @@ -100,10 +141,11 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, return nil } +// keepAliveLease maintains the lease alive by sending keep-alive requests func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { ch, err := r.client.KeepAlive(context.Background(), leaseID) if err != nil { - log.Printf("Failed to keep lease alive: %v", err) + fmt.Printf("Failed to keep lease alive: %v\n", err) return } @@ -111,12 +153,13 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { if ka != nil { fmt.Printf("Received lease keep-alive response: %v\n", ka) } else { - fmt.Printf("Lease keep-alive response channel closed") - break + fmt.Printf("Lease keep-alive response channel closed\n") + return } } } +// UnRegister removes the service endpoint from etcd func (r *SvcDiscoveryRegistryImpl) UnRegister() error { if r.endpointMgr == nil { return fmt.Errorf("endpoint manager is not initialized") @@ -124,6 +167,7 @@ func (r *SvcDiscoveryRegistryImpl) UnRegister() error { return r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) } +// Close closes the etcd client connection func (r *SvcDiscoveryRegistryImpl) Close() { if r.client != nil { _ = r.client.Close() From d2e4e2c4e5b7e61509449e7e092bfeaff8c46970 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 15:51:57 +0800 Subject: [PATCH 094/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index defec16df..e601962f5 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -211,8 +211,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C // Online push user online message to other node for _, v := range conns { - v := v - log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) + v := v // safe closure var if v.Target() == ws.disCov.GetSelfConnTarget() { log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue @@ -268,8 +267,6 @@ func (ws *WsServer) registerClient(client *Client) { } wg := sync.WaitGroup{} - log.ZDebug(client.ctx, "ws.msgGatewayConfig.Discovery.Enable", ws.msgGatewayConfig.Discovery.Enable) - if ws.msgGatewayConfig.Discovery.Enable != "k8s" { wg.Add(1) go func() { From f2757b099b1c2cdce832b2f0d4c5377a17e922f9 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 16:01:16 +0800 Subject: [PATCH 095/143] Add etcd as a service discovery mechanism --- internal/rpc/conversation/conversaion.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index ec7522212..4c7828610 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -48,10 +48,9 @@ type conversationServer struct { } type Config struct { - RpcConfig config.Conversation - RedisConfig config.Redis - MongodbConfig config.Mongo - // ZookeeperConfig config.ZooKeeper + RpcConfig config.Conversation + RedisConfig config.Redis + MongodbConfig config.Mongo NotificationConfig config.Notification Share config.Share LocalCacheConfig config.LocalCache From 3f5fcbad2b7471e36b23ad0ad9cdc96ecd11bf92 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 16:03:51 +0800 Subject: [PATCH 096/143] Add etcd as a service discovery mechanism --- internal/msgtransfer/init.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 8f72e979d..3384b8493 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -56,14 +56,13 @@ type MsgTransfer struct { } type Config struct { - MsgTransfer config.MsgTransfer - RedisConfig config.Redis - MongodbConfig config.Mongo - KafkaConfig config.Kafka - ZookeeperConfig config.ZooKeeper - Share config.Share - WebhooksConfig config.Webhooks - Discovery config.Discovery + MsgTransfer config.MsgTransfer + RedisConfig config.Redis + MongodbConfig config.Mongo + KafkaConfig config.Kafka + Share config.Share + WebhooksConfig config.Webhooks + Discovery config.Discovery } func Start(ctx context.Context, index int, config *Config) error { From 678d361ccd7555b3325ad384cf62e5229131222a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 16:44:15 +0800 Subject: [PATCH 097/143] etcd --- pkg/common/discoveryregister/etcd/etcd.go | 46 +++++++++++++++++++++++ tools/check-component/main.go | 31 ++++++++++----- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 904f83dd0..51fafd0b6 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -3,6 +3,7 @@ package etcd import ( "context" "fmt" + "github.com/pkg/errors" clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/naming/endpoints" "go.etcd.io/etcd/client/v3/naming/resolver" @@ -173,3 +174,48 @@ func (r *SvcDiscoveryRegistryImpl) Close() { _ = r.client.Close() } } + +// Check verifies if etcd is running by checking the existence of the root node and optionally creates it +func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfNotExist bool, options ...ZkOption) error { + // Configure the etcd client with default settings + cfg := clientv3.Config{ + Endpoints: etcdServers, + } + + // Apply provided options to the config + for _, opt := range options { + opt(&cfg) + } + + client, err := clientv3.New(cfg) + if err != nil { + return errors.Wrap(err, "failed to connect to etcd") + } + defer client.Close() + + // Create a child context with a default timeout or use the provided context + opCtx, cancel := context.WithCancel(ctx) + defer cancel() + + // Check if the root node exists + resp, err := client.Get(opCtx, etcdRoot) + if err != nil { + return errors.Wrap(err, "failed to get the root node from etcd") + } + + // If root node does not exist and createIfNotExist is true, create the root node + if len(resp.Kvs) == 0 { + if createIfNotExist { + _, err := client.Put(opCtx, etcdRoot, "") + if err != nil { + return errors.Wrap(err, "failed to create the root node in etcd") + } + fmt.Printf("Root node %s did not exist, but has been created.\n", etcdRoot) + } else { + return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) + } + } else { + fmt.Printf("Etcd is running and the root node %s exists.\n", etcdRoot) + } + return nil +} diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 5fa84ac36..5a936f775 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -20,9 +20,9 @@ import ( "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" - "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/minio" @@ -45,11 +45,22 @@ func CheckZookeeper(ctx context.Context, config *config.ZooKeeper) error { } func CheckEtcd(ctx context.Context, config *config.Etcd) error { - return etcd.Check(ctx, config.Address, "/check_openim_component", + etcd.Check(ctx, config.Address, "/check_openim_component", true, etcd.WithDialTimeout(10*time.Second), etcd.WithMaxCallSendMsgSize(20*1024*1024), etcd.WithUsernameAndPassword(config.Username, config.Password)) + return nil +} + +func CheckDiscovery(ctx context.Context, config *config.Discovery) error { + switch config.Enable { + case "etcd": + return CheckEtcd(ctx, &config.Etcd) + + } + + return nil } func CheckMongo(ctx context.Context, config *config.Mongo) error { @@ -139,28 +150,28 @@ func main() { func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig *config.Redis, kafkaConfig *config.Kafka, minioConfig *config.Minio, discovery *config.Discovery, maxRetry int) error { checksDone := make(map[string]bool) - checks := map[string]func(ctx context.Context) error{ - "Mongo": func(ctx context.Context) error { + checks := map[string]func() error{ + "Mongo": func() error { return CheckMongo(ctx, mongoConfig) }, - "Redis": func(ctx context.Context) error { + "Redis": func() error { return CheckRedis(ctx, redisConfig) }, - "Kafka": func(ctx context.Context) error { + "Kafka": func() error { return CheckKafka(ctx, kafkaConfig) }, } if minioConfig != nil { - checks["MinIO"] = func(ctx context.Context) error { + checks["MinIO"] = func() error { return CheckMinIO(ctx, minioConfig) } } if discovery.Enable == "etcd" { - checks["Etcd"] = func(ctx context.Context) error { + checks["Etcd"] = func() error { return CheckEtcd(ctx, &discovery.Etcd) } } else if discovery.Enable == "zookeeper" { - checks["Zookeeper"] = func(ctx context.Context) error { + checks["Zookeeper"] = func() error { return CheckZookeeper(ctx, &discovery.ZooKeeper) } } @@ -169,7 +180,7 @@ func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig * allSuccess := true for name, check := range checks { if !checksDone[name] { - if err := check(ctx); err != nil { + if err := check(); err != nil { fmt.Printf("%s check failed: %v\n", name, err) allSuccess = false } else { From 16d2bf2220f7aea5cbffb2d13a2826ac84be872d Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 16:58:49 +0800 Subject: [PATCH 098/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 12 +++++++----- tools/check-component/main.go | 13 +------------ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 51fafd0b6..eb7e6ac8b 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -193,12 +193,14 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN } defer client.Close() - // Create a child context with a default timeout or use the provided context - opCtx, cancel := context.WithCancel(ctx) - defer cancel() + if cfg.DialTimeout != 0 { + ctx, _ = context.WithTimeout(ctx, cfg.DialTimeout) + } else { + ctx, _ = context.WithTimeout(ctx, 10*time.Second) + } // Check if the root node exists - resp, err := client.Get(opCtx, etcdRoot) + resp, err := client.Get(ctx, etcdRoot) if err != nil { return errors.Wrap(err, "failed to get the root node from etcd") } @@ -206,7 +208,7 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN // If root node does not exist and createIfNotExist is true, create the root node if len(resp.Kvs) == 0 { if createIfNotExist { - _, err := client.Put(opCtx, etcdRoot, "") + _, err := client.Put(ctx, etcdRoot, "") if err != nil { return errors.Wrap(err, "failed to create the root node in etcd") } diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 5a936f775..5d4435e26 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -45,22 +45,11 @@ func CheckZookeeper(ctx context.Context, config *config.ZooKeeper) error { } func CheckEtcd(ctx context.Context, config *config.Etcd) error { - etcd.Check(ctx, config.Address, "/check_openim_component", + return etcd.Check(ctx, config.Address, "/check_openim_component", true, etcd.WithDialTimeout(10*time.Second), etcd.WithMaxCallSendMsgSize(20*1024*1024), etcd.WithUsernameAndPassword(config.Username, config.Password)) - return nil -} - -func CheckDiscovery(ctx context.Context, config *config.Discovery) error { - switch config.Enable { - case "etcd": - return CheckEtcd(ctx, &config.Etcd) - - } - - return nil } func CheckMongo(ctx context.Context, config *config.Mongo) error { From 54d5ff267ba615dcf249a748292fe95c8d0006e2 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 17:01:35 +0800 Subject: [PATCH 099/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index eb7e6ac8b..6866186b0 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -212,7 +212,6 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN if err != nil { return errors.Wrap(err, "failed to create the root node in etcd") } - fmt.Printf("Root node %s did not exist, but has been created.\n", etcdRoot) } else { return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) } From 101430a3002d8fe4c23d0d44f5aabadde592b1b3 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 17:02:38 +0800 Subject: [PATCH 100/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 6866186b0..1f952bbf7 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -152,7 +152,6 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { for ka := range ch { if ka != nil { - fmt.Printf("Received lease keep-alive response: %v\n", ka) } else { fmt.Printf("Lease keep-alive response channel closed\n") return From 4c3224ce5aaca008c8e02cabbf456db5212d41fa Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 17:19:00 +0800 Subject: [PATCH 101/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 34 +++++++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 1f952bbf7..d9eb86aa3 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -174,7 +174,7 @@ func (r *SvcDiscoveryRegistryImpl) Close() { } } -// Check verifies if etcd is running by checking the existence of the root node and optionally creates it +// Check verifies if etcd is running by checking the existence of the root node and optionally creates it with a lease func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfNotExist bool, options ...ZkOption) error { // Configure the etcd client with default settings cfg := clientv3.Config{ @@ -192,30 +192,52 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN } defer client.Close() + // Determine timeout for context + var opCtx context.Context + var cancel context.CancelFunc if cfg.DialTimeout != 0 { - ctx, _ = context.WithTimeout(ctx, cfg.DialTimeout) + opCtx, cancel = context.WithTimeout(ctx, cfg.DialTimeout) } else { - ctx, _ = context.WithTimeout(ctx, 10*time.Second) + opCtx, cancel = context.WithTimeout(ctx, 10*time.Second) } + defer cancel() // Check if the root node exists - resp, err := client.Get(ctx, etcdRoot) + resp, err := client.Get(opCtx, etcdRoot) if err != nil { return errors.Wrap(err, "failed to get the root node from etcd") } - // If root node does not exist and createIfNotExist is true, create the root node + // If root node does not exist and createIfNotExist is true, create the root node with a lease if len(resp.Kvs) == 0 { if createIfNotExist { - _, err := client.Put(ctx, etcdRoot, "") + var leaseTTL int64 = 10 + var leaseResp *clientv3.LeaseGrantResponse + if leaseTTL > 0 { + // Create a lease + leaseResp, err = client.Grant(opCtx, leaseTTL) + if err != nil { + return errors.Wrap(err, "failed to create lease in etcd") + } + } + + // Put the key with the lease + putOpts := []clientv3.OpOption{} + if leaseResp != nil { + putOpts = append(putOpts, clientv3.WithLease(leaseResp.ID)) + } + + _, err := client.Put(opCtx, etcdRoot, "", putOpts...) if err != nil { return errors.Wrap(err, "failed to create the root node in etcd") } + fmt.Printf("Root node %s did not exist, but has been created.\n", etcdRoot) } else { return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) } } else { fmt.Printf("Etcd is running and the root node %s exists.\n", etcdRoot) } + return nil } From 7e3722c9c463f225ca20f35298c5bcc55ad04469 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 17:39:15 +0800 Subject: [PATCH 102/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index d9eb86aa3..741ceb8b1 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -85,7 +85,7 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, // GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - target := fmt.Sprintf("etcd:///%s", serviceName) + target := fmt.Sprintf("etcd:///%s/%s", r.rootDirectory, serviceName) conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { return nil, err @@ -95,7 +95,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str // GetConn returns a single gRPC client connection for a given service name func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - target := fmt.Sprintf("etcd:///%s", serviceName) + target := fmt.Sprintf("etcd:///%s/%s", r.rootDirectory, serviceName) return grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) } From 676320454a0511221196570a3517a3bafa2c1c92 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 18:06:54 +0800 Subject: [PATCH 103/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 741ceb8b1..1707bf776 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -17,12 +17,14 @@ type ZkOption func(*clientv3.Config) // SvcDiscoveryRegistryImpl implementation type SvcDiscoveryRegistryImpl struct { - client *clientv3.Client - resolver gresolver.Builder - dialOptions []grpc.DialOption - serviceKey string - endpointMgr endpoints.Manager - leaseID clientv3.LeaseID + client *clientv3.Client + resolver gresolver.Builder + dialOptions []grpc.DialOption + serviceKey string + endpointMgr endpoints.Manager + leaseID clientv3.LeaseID + rpcRegisterTarget string + rootDirectory string } @@ -101,7 +103,9 @@ func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName stri // GetSelfConnTarget returns the connection target for the current service func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { - return fmt.Sprintf("etcd:///%s", r.serviceKey) + return r.rpcRegisterTarget + // return fmt.Sprintf("etcd:///%s", r.serviceKey) + } // AddOption appends gRPC dial options to the existing options @@ -131,7 +135,9 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, } r.leaseID = leaseResp.ID - endpoint := endpoints.Endpoint{Addr: fmt.Sprintf("%s:%d", host, port)} + r.rpcRegisterTarget = fmt.Sprintf("%s:%d", host, port) + endpoint := endpoints.Endpoint{Addr: r.rpcRegisterTarget} + err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) if err != nil { return err From 21999fc546cdb05a7c542013ede6e62d81c09b84 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 18:09:32 +0800 Subject: [PATCH 104/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index e601962f5..fc9f88d59 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,7 +216,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - + log.ZDebug(ctx, "target ", v.Target()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) _, err := msgClient.MultiTerminalLoginCheck(ctx, &msggateway.MultiTerminalLoginCheckReq{ From 4dc0c3d0d68c23994284a56419102734615d1e91 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 18:21:26 +0800 Subject: [PATCH 105/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index fc9f88d59..3ed5e48a6 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,7 +216,9 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } + fmt.Printf("Connection target: %s\n", v.Target()) log.ZDebug(ctx, "target ", v.Target()) + wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) _, err := msgClient.MultiTerminalLoginCheck(ctx, &msggateway.MultiTerminalLoginCheckReq{ From eccdaf74a8b7fdff1e0778b4a1797bfe583eb1df Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Sat, 11 May 2024 18:26:47 +0800 Subject: [PATCH 106/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 3ed5e48a6..2e38c1720 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,8 +216,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - fmt.Printf("Connection target: %s\n", v.Target()) - log.ZDebug(ctx, "target ", v.Target()) + log.ZDebug(ctx, "conn ", "target", v.Target()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) From 920a89070aa680f3a4ff098a698b325b821dbfec Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 10:06:17 +0800 Subject: [PATCH 107/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 2e38c1720..749fa40b5 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,7 +216,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - log.ZDebug(ctx, "conn ", "target", v.Target()) + log.ZDebug(ctx, "sendUserOnlineInfoToOtherNode conn ", "target", v.Target(), v.GetState()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) From 76ef4f9b40f7eef7000ac80a7a7484384cac73dd Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 10:06:36 +0800 Subject: [PATCH 108/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 749fa40b5..6ef2c3213 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -216,7 +216,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - log.ZDebug(ctx, "sendUserOnlineInfoToOtherNode conn ", "target", v.Target(), v.GetState()) + log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target(), v.GetState()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) From e44f0531007606b68896ee163b52e1601bf501c5 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 10:19:24 +0800 Subject: [PATCH 109/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 6ef2c3213..29cbb51d8 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -211,12 +211,11 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C // Online push user online message to other node for _, v := range conns { - v := v // safe closure var + log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) if v.Target() == ws.disCov.GetSelfConnTarget() { log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } - log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target(), v.GetState()) wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(v) From 3074604c86aea6f9a62f6f467fc88394b2d1aaf8 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 10:22:41 +0800 Subject: [PATCH 110/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 29cbb51d8..a29b58528 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -212,8 +212,11 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C // Online push user online message to other node for _, v := range conns { log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) - if v.Target() == ws.disCov.GetSelfConnTarget() { - log.ZDebug(ctx, "Filter out this node", "node", v.Target()) + //if v.Target() == ws.disCov.GetSelfConnTarget() { + // log.ZDebug(ctx, "Filter out this node", "node", v.Target()) + // continue + //} + if true { continue } From f51cdadb57de8f55d905787f7533ab2b6679c4ba Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 11:10:25 +0800 Subject: [PATCH 111/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 1707bf776..1950223a1 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -87,12 +87,27 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, // GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - target := fmt.Sprintf("etcd:///%s/%s", r.rootDirectory, serviceName) - conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) + var conns []*grpc.ClientConn + // Construct the full key for the service + fullServiceKey := fmt.Sprintf("%s/%s", r.rootDirectory, serviceName) + + // List all endpoints for the service + resp, err := r.client.Get(ctx, fullServiceKey, clientv3.WithPrefix()) if err != nil { return nil, err } - return []*grpc.ClientConn{conn}, nil + + for _, kv := range resp.Kvs { + endpoint := string(kv.Key[len(fullServiceKey)+1:]) // Extract the endpoint address + target := fmt.Sprintf("etcd://%s", endpoint) + conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) + if err != nil { + return nil, err + } + conns = append(conns, conn) + } + + return conns, nil } // GetConn returns a single gRPC client connection for a given service name From 6041f4e79398842961c70b16d8017598d3a66b38 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 11:10:42 +0800 Subject: [PATCH 112/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index a29b58528..29cbb51d8 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -212,11 +212,8 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C // Online push user online message to other node for _, v := range conns { log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) - //if v.Target() == ws.disCov.GetSelfConnTarget() { - // log.ZDebug(ctx, "Filter out this node", "node", v.Target()) - // continue - //} - if true { + if v.Target() == ws.disCov.GetSelfConnTarget() { + log.ZDebug(ctx, "Filter out this node", "node", v.Target()) continue } From 4d300e9abef5099a0ac0f42b9e86516ca0877707 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 11:50:01 +0800 Subject: [PATCH 113/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 29cbb51d8..515e1f7e8 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -267,6 +267,8 @@ func (ws *WsServer) registerClient(client *Client) { } wg := sync.WaitGroup{} + log.ZDebug(client.ctx, "ws.msgGatewayConfig.Discovery.Enable", ws.msgGatewayConfig.Discovery.Enable) + if ws.msgGatewayConfig.Discovery.Enable != "k8s" { wg.Add(1) go func() { From ac7e074d612c519bcbb38556cd77abd58a4da592 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 12:15:35 +0800 Subject: [PATCH 114/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 1950223a1..af827e4b7 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -94,6 +94,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str // List all endpoints for the service resp, err := r.client.Get(ctx, fullServiceKey, clientv3.WithPrefix()) if err != nil { + fmt.Println("GetConns get ", fullServiceKey, err.Error()) return nil, err } @@ -102,6 +103,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str target := fmt.Sprintf("etcd://%s", endpoint) conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { + fmt.Println("DialContext ", target, err.Error()) return nil, err } conns = append(conns, conn) From fd25946b1c39c3f81f0d56951eef7530a8b83900 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 12:17:56 +0800 Subject: [PATCH 115/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index af827e4b7..5d0e37d0f 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -100,7 +100,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str for _, kv := range resp.Kvs { endpoint := string(kv.Key[len(fullServiceKey)+1:]) // Extract the endpoint address - target := fmt.Sprintf("etcd://%s", endpoint) + target := fmt.Sprintf("etcd://%s/%s/%s", r.rootDirectory, serviceName, endpoint) conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { fmt.Println("DialContext ", target, err.Error()) From a546f215565e34e91d5c49e57e21ee54d70a574f Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 14:08:35 +0800 Subject: [PATCH 116/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 5d0e37d0f..bf510051f 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -108,7 +108,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str } conns = append(conns, conn) } - + fmt.Println("GetConns number ", len(conns)) return conns, nil } From 5a976e3ef7abac9a1a760d757fce472af4cd547b Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 15:04:59 +0800 Subject: [PATCH 117/143] Add etcd as a service discovery mechanism --- internal/msggateway/n_ws_server.go | 1 + internal/push/onlinepusher.go | 1 + pkg/common/discoveryregister/etcd/etcd.go | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/msggateway/n_ws_server.go b/internal/msggateway/n_ws_server.go index 515e1f7e8..defec16df 100644 --- a/internal/msggateway/n_ws_server.go +++ b/internal/msggateway/n_ws_server.go @@ -211,6 +211,7 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C // Online push user online message to other node for _, v := range conns { + v := v log.ZDebug(ctx, " sendUserOnlineInfoToOtherNode conn ", "target", v.Target()) if v.Target() == ws.disCov.GetSelfConnTarget() { log.ZDebug(ctx, "Filter out this node", "node", v.Target()) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index a61399fb6..adeff3737 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -86,6 +86,7 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M // Online push message for _, conn := range conns { + log.ZDebug(ctx, "get gateway conn detail ", "conn ", *conn) conn := conn // loop var safe ctx := ctx wg.Go(func() error { diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index bf510051f..7baac73f3 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -107,8 +107,9 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str return nil, err } conns = append(conns, conn) + fmt.Println("GetConns detail ", *conn) + } - fmt.Println("GetConns number ", len(conns)) return conns, nil } From 817686754375742b60fbb3a1463f43db961de069 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 15:07:40 +0800 Subject: [PATCH 118/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index adeff3737..84b7f8af3 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -2,6 +2,7 @@ package push import ( "context" + "fmt" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" @@ -86,7 +87,8 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M // Online push message for _, conn := range conns { - log.ZDebug(ctx, "get gateway conn detail ", "conn ", *conn) + fmt.Println(ctx, "get gateway conn detail ", "conn ", *conn) + conn := conn // loop var safe ctx := ctx wg.Go(func() error { From 4d0d9bb022c118623287bd1731f052ae09488d33 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 15:22:17 +0800 Subject: [PATCH 119/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 7baac73f3..2c4385f68 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -100,7 +100,8 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str for _, kv := range resp.Kvs { endpoint := string(kv.Key[len(fullServiceKey)+1:]) // Extract the endpoint address - target := fmt.Sprintf("etcd://%s/%s/%s", r.rootDirectory, serviceName, endpoint) + //target := fmt.Sprintf("etcd://%s/%s/%s", r.rootDirectory, serviceName, endpoint) + target := endpoint conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) if err != nil { fmt.Println("DialContext ", target, err.Error()) @@ -140,7 +141,7 @@ func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { // Register registers a new service endpoint with etcd func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s/%s-%d", r.rootDirectory, serviceName, host, port) + r.serviceKey = fmt.Sprintf("%s/%s/%s:%d", r.rootDirectory, serviceName, host, port) em, err := endpoints.NewManager(r.client, r.rootDirectory+"/"+serviceName) if err != nil { return err From f5b117e651ecca93866c7ffefce93e2094f10f8a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 17:20:15 +0800 Subject: [PATCH 120/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 105 ++++++++++++++-------- 1 file changed, 69 insertions(+), 36 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 2c4385f68..7c2bcff66 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -9,6 +9,8 @@ import ( "go.etcd.io/etcd/client/v3/naming/resolver" "google.golang.org/grpc" gresolver "google.golang.org/grpc/resolver" + "strings" + "sync" "time" ) @@ -26,6 +28,9 @@ type SvcDiscoveryRegistryImpl struct { rpcRegisterTarget string rootDirectory string + + mu sync.RWMutex + connMap map[string][]*grpc.ClientConn } // NewSvcDiscoveryRegistry creates a new service discovery registry implementation @@ -51,11 +56,15 @@ func NewSvcDiscoveryRegistry(rootDirectory string, endpoints []string, options . if err != nil { return nil, err } - return &SvcDiscoveryRegistryImpl{ + + s := &SvcDiscoveryRegistryImpl{ client: client, resolver: r, rootDirectory: rootDirectory, - }, nil + connMap: make(map[string][]*grpc.ClientConn), + } + go s.watchServiceChanges() + return s, nil } // WithDialTimeout sets a custom dial timeout for the etcd client @@ -87,31 +96,11 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, // GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - var conns []*grpc.ClientConn - // Construct the full key for the service fullServiceKey := fmt.Sprintf("%s/%s", r.rootDirectory, serviceName) - // List all endpoints for the service - resp, err := r.client.Get(ctx, fullServiceKey, clientv3.WithPrefix()) - if err != nil { - fmt.Println("GetConns get ", fullServiceKey, err.Error()) - return nil, err - } - - for _, kv := range resp.Kvs { - endpoint := string(kv.Key[len(fullServiceKey)+1:]) // Extract the endpoint address - //target := fmt.Sprintf("etcd://%s/%s/%s", r.rootDirectory, serviceName, endpoint) - target := endpoint - conn, err := grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) - if err != nil { - fmt.Println("DialContext ", target, err.Error()) - return nil, err - } - conns = append(conns, conn) - fmt.Println("GetConns detail ", *conn) - - } - return conns, nil + r.mu.RLock() + defer r.mu.RUnlock() + return r.connMap[fullServiceKey], nil } // GetConn returns a single gRPC client connection for a given service name @@ -123,8 +112,6 @@ func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName stri // GetSelfConnTarget returns the connection target for the current service func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { return r.rpcRegisterTarget - // return fmt.Sprintf("etcd:///%s", r.serviceKey) - } // AddOption appends gRPC dial options to the existing options @@ -163,7 +150,6 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, } go r.keepAliveLease(r.leaseID) - return nil } @@ -184,12 +170,63 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { } } +// watchServiceChanges watches for changes in the service directory +func (r *SvcDiscoveryRegistryImpl) watchServiceChanges() { + watchChan := r.client.Watch(context.Background(), r.rootDirectory, clientv3.WithPrefix()) + for watchResp := range watchChan { + for _, event := range watchResp.Events { + prefix, _ := r.splitEndpoint(string(event.Kv.Key)) + fmt.Printf("Change detected for prefix: %s\n", prefix) + r.refreshConnMap(prefix) + } + } +} + +// refreshConnMap fetches the latest endpoints and updates the local map +func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { + r.mu.Lock() + defer r.mu.Unlock() + + fullPrefix := fmt.Sprintf("%s/", prefix) + resp, err := r.client.Get(context.Background(), fullPrefix, clientv3.WithPrefix()) + if err != nil { + fmt.Printf("Failed to get endpoints: %v\n", err) + return + } + + // Update the connMap with new connections + for _, kv := range resp.Kvs { + _, addr := r.splitEndpoint(string(kv.Key)) + conn, err := grpc.DialContext(context.Background(), addr, append(r.dialOptions, grpc.WithResolvers(r.resolver))...) + if err != nil { + fmt.Printf("Failed to dial new endpoint: %v\n", err) + continue + } + r.connMap[prefix] = append(r.connMap[prefix], conn) + } +} + +// splitEndpoint splits the endpoint string into prefix and address +func (r *SvcDiscoveryRegistryImpl) splitEndpoint(input string) (string, string) { + lastSlashIndex := strings.LastIndex(input, "/") + if lastSlashIndex != -1 { + part1 := input[:lastSlashIndex] + part2 := input[lastSlashIndex+1:] + return part1, part2 + } + return input, "" +} + // UnRegister removes the service endpoint from etcd func (r *SvcDiscoveryRegistryImpl) UnRegister() error { if r.endpointMgr == nil { return fmt.Errorf("endpoint manager is not initialized") } - return r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) + err := r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) + if err != nil { + return err + } + return nil } // Close closes the etcd client connection @@ -197,16 +234,17 @@ func (r *SvcDiscoveryRegistryImpl) Close() { if r.client != nil { _ = r.client.Close() } + + r.mu.Lock() + defer r.mu.Unlock() } // Check verifies if etcd is running by checking the existence of the root node and optionally creates it with a lease func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfNotExist bool, options ...ZkOption) error { - // Configure the etcd client with default settings cfg := clientv3.Config{ Endpoints: etcdServers, } - // Apply provided options to the config for _, opt := range options { opt(&cfg) } @@ -217,7 +255,6 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN } defer client.Close() - // Determine timeout for context var opCtx context.Context var cancel context.CancelFunc if cfg.DialTimeout != 0 { @@ -227,26 +264,22 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN } defer cancel() - // Check if the root node exists resp, err := client.Get(opCtx, etcdRoot) if err != nil { return errors.Wrap(err, "failed to get the root node from etcd") } - // If root node does not exist and createIfNotExist is true, create the root node with a lease if len(resp.Kvs) == 0 { if createIfNotExist { var leaseTTL int64 = 10 var leaseResp *clientv3.LeaseGrantResponse if leaseTTL > 0 { - // Create a lease leaseResp, err = client.Grant(opCtx, leaseTTL) if err != nil { return errors.Wrap(err, "failed to create lease in etcd") } } - // Put the key with the lease putOpts := []clientv3.OpOption{} if leaseResp != nil { putOpts = append(putOpts, clientv3.WithLease(leaseResp.ID)) From 0fed297615a2b6d761ea7825568a93a24e5ea26b Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 17:44:40 +0800 Subject: [PATCH 121/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 7c2bcff66..4ce60dbd5 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -193,8 +193,7 @@ func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { fmt.Printf("Failed to get endpoints: %v\n", err) return } - - // Update the connMap with new connections + r.connMap[prefix] = []*grpc.ClientConn{} // Update the connMap with new connections for _, kv := range resp.Kvs { _, addr := r.splitEndpoint(string(kv.Key)) conn, err := grpc.DialContext(context.Background(), addr, append(r.dialOptions, grpc.WithResolvers(r.resolver))...) From 3d4e95d7c8546bf022ae5c88f4c392f7d76fc57a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Mon, 13 May 2024 17:50:40 +0800 Subject: [PATCH 122/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/etcd/etcd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 4ce60dbd5..247161f9b 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -100,6 +100,7 @@ func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName str r.mu.RLock() defer r.mu.RUnlock() + fmt.Printf("all conns ", serviceName, r.connMap[fullServiceKey]) return r.connMap[fullServiceKey], nil } From 73824cc5f28df868f0172d1f440935a0df7f6769 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 08:57:55 +0800 Subject: [PATCH 123/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 3 --- pkg/common/discoveryregister/etcd/etcd.go | 28 +++++++---------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 84b7f8af3..a61399fb6 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -2,7 +2,6 @@ package push import ( "context" - "fmt" "github.com/openimsdk/protocol/msggateway" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/discovery" @@ -87,8 +86,6 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M // Online push message for _, conn := range conns { - fmt.Println(ctx, "get gateway conn detail ", "conn ", *conn) - conn := conn // loop var safe ctx := ctx wg.Go(func() error { diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go index 247161f9b..dd8806378 100644 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ b/pkg/common/discoveryregister/etcd/etcd.go @@ -97,10 +97,8 @@ func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, // GetConns returns gRPC client connections for a given service name func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { fullServiceKey := fmt.Sprintf("%s/%s", r.rootDirectory, serviceName) - r.mu.RLock() defer r.mu.RUnlock() - fmt.Printf("all conns ", serviceName, r.connMap[fullServiceKey]) return r.connMap[fullServiceKey], nil } @@ -122,9 +120,7 @@ func (r *SvcDiscoveryRegistryImpl) AddOption(opts ...grpc.DialOption) { // CloseConn closes a given gRPC client connection func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { - if err := conn.Close(); err != nil { - fmt.Printf("Failed to close connection: %v\n", err) - } + conn.Close() } // Register registers a new service endpoint with etcd @@ -136,7 +132,7 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, } r.endpointMgr = em - leaseResp, err := r.client.Grant(context.Background(), 60) // Increase TTL time + leaseResp, err := r.client.Grant(context.Background(), 30) // if err != nil { return err } @@ -158,14 +154,11 @@ func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { ch, err := r.client.KeepAlive(context.Background(), leaseID) if err != nil { - fmt.Printf("Failed to keep lease alive: %v\n", err) return } - for ka := range ch { if ka != nil { } else { - fmt.Printf("Lease keep-alive response channel closed\n") return } } @@ -175,10 +168,14 @@ func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { func (r *SvcDiscoveryRegistryImpl) watchServiceChanges() { watchChan := r.client.Watch(context.Background(), r.rootDirectory, clientv3.WithPrefix()) for watchResp := range watchChan { + updatedPrefixes := make(map[string]struct{}) // Create a set to track updated prefixes + for _, event := range watchResp.Events { prefix, _ := r.splitEndpoint(string(event.Kv.Key)) - fmt.Printf("Change detected for prefix: %s\n", prefix) - r.refreshConnMap(prefix) + if _, alreadyUpdated := updatedPrefixes[prefix]; !alreadyUpdated { + updatedPrefixes[prefix] = struct{}{} // Mark this prefix as updated + r.refreshConnMap(prefix) + } } } } @@ -191,7 +188,6 @@ func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { fullPrefix := fmt.Sprintf("%s/", prefix) resp, err := r.client.Get(context.Background(), fullPrefix, clientv3.WithPrefix()) if err != nil { - fmt.Printf("Failed to get endpoints: %v\n", err) return } r.connMap[prefix] = []*grpc.ClientConn{} // Update the connMap with new connections @@ -199,7 +195,6 @@ func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { _, addr := r.splitEndpoint(string(kv.Key)) conn, err := grpc.DialContext(context.Background(), addr, append(r.dialOptions, grpc.WithResolvers(r.resolver))...) if err != nil { - fmt.Printf("Failed to dial new endpoint: %v\n", err) continue } r.connMap[prefix] = append(r.connMap[prefix], conn) @@ -244,11 +239,9 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN cfg := clientv3.Config{ Endpoints: etcdServers, } - for _, opt := range options { opt(&cfg) } - client, err := clientv3.New(cfg) if err != nil { return errors.Wrap(err, "failed to connect to etcd") @@ -279,7 +272,6 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN return errors.Wrap(err, "failed to create lease in etcd") } } - putOpts := []clientv3.OpOption{} if leaseResp != nil { putOpts = append(putOpts, clientv3.WithLease(leaseResp.ID)) @@ -289,13 +281,9 @@ func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfN if err != nil { return errors.Wrap(err, "failed to create the root node in etcd") } - fmt.Printf("Root node %s did not exist, but has been created.\n", etcdRoot) } else { return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) } - } else { - fmt.Printf("Etcd is running and the root node %s exists.\n", etcdRoot) } - return nil } From f16b3c50bd8b7886b5b0239684ecfe7560bbe810 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 10:42:24 +0800 Subject: [PATCH 124/143] Add etcd as a service discovery mechanism --- go.mod | 4 +- go.sum | 4 +- .../discoveryregister/discoveryregister.go | 2 +- pkg/common/discoveryregister/etcd/etcd.go | 289 ------------------ 4 files changed, 5 insertions(+), 294 deletions(-) delete mode 100644 pkg/common/discoveryregister/etcd/etcd.go diff --git a/go.mod b/go.mod index 2ec3e20b2..640b4a824 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.9 + github.com/openimsdk/tools v0.0.49-alpha.10 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 @@ -40,7 +40,6 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.18.2 github.com/stathat/consistent v1.0.0 - go.etcd.io/etcd/client/v3 v3.5.13 go.uber.org/automaxprocs v1.5.3 golang.org/x/sync v0.6.0 ) @@ -142,6 +141,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/etcd/api/v3 v3.5.13 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect + go.etcd.io/etcd/client/v3 v3.5.13 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect diff --git a/go.sum b/go.sum index edecead94..241bb96aa 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.9 h1:yoa3GS6t0d1mRv/S86niFBGDgSjy2EWWwBI5NAH1Kgk= -github.com/openimsdk/tools v0.0.49-alpha.9/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.10 h1:vcfMUGBClD3TsjsTd/Wb0R2WcpjCo8hupMqreb0dXFA= +github.com/openimsdk/tools v0.0.49-alpha.10/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 1085ec1ea..f37fc9c10 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -16,9 +16,9 @@ package discoveryregister import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" - getcd "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" + getcd "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/errs" "time" diff --git a/pkg/common/discoveryregister/etcd/etcd.go b/pkg/common/discoveryregister/etcd/etcd.go deleted file mode 100644 index dd8806378..000000000 --- a/pkg/common/discoveryregister/etcd/etcd.go +++ /dev/null @@ -1,289 +0,0 @@ -package etcd - -import ( - "context" - "fmt" - "github.com/pkg/errors" - clientv3 "go.etcd.io/etcd/client/v3" - "go.etcd.io/etcd/client/v3/naming/endpoints" - "go.etcd.io/etcd/client/v3/naming/resolver" - "google.golang.org/grpc" - gresolver "google.golang.org/grpc/resolver" - "strings" - "sync" - "time" -) - -// ZkOption defines a function type for modifying clientv3.Config -type ZkOption func(*clientv3.Config) - -// SvcDiscoveryRegistryImpl implementation -type SvcDiscoveryRegistryImpl struct { - client *clientv3.Client - resolver gresolver.Builder - dialOptions []grpc.DialOption - serviceKey string - endpointMgr endpoints.Manager - leaseID clientv3.LeaseID - rpcRegisterTarget string - - rootDirectory string - - mu sync.RWMutex - connMap map[string][]*grpc.ClientConn -} - -// NewSvcDiscoveryRegistry creates a new service discovery registry implementation -func NewSvcDiscoveryRegistry(rootDirectory string, endpoints []string, options ...ZkOption) (*SvcDiscoveryRegistryImpl, error) { - cfg := clientv3.Config{ - Endpoints: endpoints, - DialTimeout: 5 * time.Second, - // Increase keep-alive queue capacity and message size - PermitWithoutStream: true, - MaxCallSendMsgSize: 10 * 1024 * 1024, // 10 MB - } - - // Apply provided options to the config - for _, opt := range options { - opt(&cfg) - } - - client, err := clientv3.New(cfg) - if err != nil { - return nil, err - } - r, err := resolver.NewBuilder(client) - if err != nil { - return nil, err - } - - s := &SvcDiscoveryRegistryImpl{ - client: client, - resolver: r, - rootDirectory: rootDirectory, - connMap: make(map[string][]*grpc.ClientConn), - } - go s.watchServiceChanges() - return s, nil -} - -// WithDialTimeout sets a custom dial timeout for the etcd client -func WithDialTimeout(timeout time.Duration) ZkOption { - return func(cfg *clientv3.Config) { - cfg.DialTimeout = timeout - } -} - -// WithMaxCallSendMsgSize sets a custom max call send message size for the etcd client -func WithMaxCallSendMsgSize(size int) ZkOption { - return func(cfg *clientv3.Config) { - cfg.MaxCallSendMsgSize = size - } -} - -// WithUsernameAndPassword sets a username and password for the etcd client -func WithUsernameAndPassword(username, password string) ZkOption { - return func(cfg *clientv3.Config) { - cfg.Username = username - cfg.Password = password - } -} - -// GetUserIdHashGatewayHost returns the gateway host for a given user ID hash -func (r *SvcDiscoveryRegistryImpl) GetUserIdHashGatewayHost(ctx context.Context, userId string) (string, error) { - return "", nil -} - -// GetConns returns gRPC client connections for a given service name -func (r *SvcDiscoveryRegistryImpl) GetConns(ctx context.Context, serviceName string, opts ...grpc.DialOption) ([]*grpc.ClientConn, error) { - fullServiceKey := fmt.Sprintf("%s/%s", r.rootDirectory, serviceName) - r.mu.RLock() - defer r.mu.RUnlock() - return r.connMap[fullServiceKey], nil -} - -// GetConn returns a single gRPC client connection for a given service name -func (r *SvcDiscoveryRegistryImpl) GetConn(ctx context.Context, serviceName string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - target := fmt.Sprintf("etcd:///%s/%s", r.rootDirectory, serviceName) - return grpc.DialContext(ctx, target, append(append(r.dialOptions, opts...), grpc.WithResolvers(r.resolver))...) -} - -// GetSelfConnTarget returns the connection target for the current service -func (r *SvcDiscoveryRegistryImpl) GetSelfConnTarget() string { - return r.rpcRegisterTarget -} - -// AddOption appends gRPC dial options to the existing options -func (r *SvcDiscoveryRegistryImpl) AddOption(opts ...grpc.DialOption) { - r.dialOptions = append(r.dialOptions, opts...) -} - -// CloseConn closes a given gRPC client connection -func (r *SvcDiscoveryRegistryImpl) CloseConn(conn *grpc.ClientConn) { - conn.Close() -} - -// Register registers a new service endpoint with etcd -func (r *SvcDiscoveryRegistryImpl) Register(serviceName, host string, port int, opts ...grpc.DialOption) error { - r.serviceKey = fmt.Sprintf("%s/%s/%s:%d", r.rootDirectory, serviceName, host, port) - em, err := endpoints.NewManager(r.client, r.rootDirectory+"/"+serviceName) - if err != nil { - return err - } - r.endpointMgr = em - - leaseResp, err := r.client.Grant(context.Background(), 30) // - if err != nil { - return err - } - r.leaseID = leaseResp.ID - - r.rpcRegisterTarget = fmt.Sprintf("%s:%d", host, port) - endpoint := endpoints.Endpoint{Addr: r.rpcRegisterTarget} - - err = em.AddEndpoint(context.TODO(), r.serviceKey, endpoint, clientv3.WithLease(leaseResp.ID)) - if err != nil { - return err - } - - go r.keepAliveLease(r.leaseID) - return nil -} - -// keepAliveLease maintains the lease alive by sending keep-alive requests -func (r *SvcDiscoveryRegistryImpl) keepAliveLease(leaseID clientv3.LeaseID) { - ch, err := r.client.KeepAlive(context.Background(), leaseID) - if err != nil { - return - } - for ka := range ch { - if ka != nil { - } else { - return - } - } -} - -// watchServiceChanges watches for changes in the service directory -func (r *SvcDiscoveryRegistryImpl) watchServiceChanges() { - watchChan := r.client.Watch(context.Background(), r.rootDirectory, clientv3.WithPrefix()) - for watchResp := range watchChan { - updatedPrefixes := make(map[string]struct{}) // Create a set to track updated prefixes - - for _, event := range watchResp.Events { - prefix, _ := r.splitEndpoint(string(event.Kv.Key)) - if _, alreadyUpdated := updatedPrefixes[prefix]; !alreadyUpdated { - updatedPrefixes[prefix] = struct{}{} // Mark this prefix as updated - r.refreshConnMap(prefix) - } - } - } -} - -// refreshConnMap fetches the latest endpoints and updates the local map -func (r *SvcDiscoveryRegistryImpl) refreshConnMap(prefix string) { - r.mu.Lock() - defer r.mu.Unlock() - - fullPrefix := fmt.Sprintf("%s/", prefix) - resp, err := r.client.Get(context.Background(), fullPrefix, clientv3.WithPrefix()) - if err != nil { - return - } - r.connMap[prefix] = []*grpc.ClientConn{} // Update the connMap with new connections - for _, kv := range resp.Kvs { - _, addr := r.splitEndpoint(string(kv.Key)) - conn, err := grpc.DialContext(context.Background(), addr, append(r.dialOptions, grpc.WithResolvers(r.resolver))...) - if err != nil { - continue - } - r.connMap[prefix] = append(r.connMap[prefix], conn) - } -} - -// splitEndpoint splits the endpoint string into prefix and address -func (r *SvcDiscoveryRegistryImpl) splitEndpoint(input string) (string, string) { - lastSlashIndex := strings.LastIndex(input, "/") - if lastSlashIndex != -1 { - part1 := input[:lastSlashIndex] - part2 := input[lastSlashIndex+1:] - return part1, part2 - } - return input, "" -} - -// UnRegister removes the service endpoint from etcd -func (r *SvcDiscoveryRegistryImpl) UnRegister() error { - if r.endpointMgr == nil { - return fmt.Errorf("endpoint manager is not initialized") - } - err := r.endpointMgr.DeleteEndpoint(context.TODO(), r.serviceKey) - if err != nil { - return err - } - return nil -} - -// Close closes the etcd client connection -func (r *SvcDiscoveryRegistryImpl) Close() { - if r.client != nil { - _ = r.client.Close() - } - - r.mu.Lock() - defer r.mu.Unlock() -} - -// Check verifies if etcd is running by checking the existence of the root node and optionally creates it with a lease -func Check(ctx context.Context, etcdServers []string, etcdRoot string, createIfNotExist bool, options ...ZkOption) error { - cfg := clientv3.Config{ - Endpoints: etcdServers, - } - for _, opt := range options { - opt(&cfg) - } - client, err := clientv3.New(cfg) - if err != nil { - return errors.Wrap(err, "failed to connect to etcd") - } - defer client.Close() - - var opCtx context.Context - var cancel context.CancelFunc - if cfg.DialTimeout != 0 { - opCtx, cancel = context.WithTimeout(ctx, cfg.DialTimeout) - } else { - opCtx, cancel = context.WithTimeout(ctx, 10*time.Second) - } - defer cancel() - - resp, err := client.Get(opCtx, etcdRoot) - if err != nil { - return errors.Wrap(err, "failed to get the root node from etcd") - } - - if len(resp.Kvs) == 0 { - if createIfNotExist { - var leaseTTL int64 = 10 - var leaseResp *clientv3.LeaseGrantResponse - if leaseTTL > 0 { - leaseResp, err = client.Grant(opCtx, leaseTTL) - if err != nil { - return errors.Wrap(err, "failed to create lease in etcd") - } - } - putOpts := []clientv3.OpOption{} - if leaseResp != nil { - putOpts = append(putOpts, clientv3.WithLease(leaseResp.ID)) - } - - _, err := client.Put(opCtx, etcdRoot, "", putOpts...) - if err != nil { - return errors.Wrap(err, "failed to create the root node in etcd") - } - } else { - return fmt.Errorf("root node %s does not exist in etcd", etcdRoot) - } - } - return nil -} From 2d1f08c99406aef2c862563c740c44d8b2fea702 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 10:52:08 +0800 Subject: [PATCH 125/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/discoveryregister.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index f37fc9c10..1debc54e1 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -18,7 +18,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/kubernetes" "github.com/openimsdk/tools/discovery" - getcd "github.com/openimsdk/tools/discovery/etcd" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/errs" "time" @@ -39,12 +39,12 @@ func NewDiscoveryRegister(discovery *config.Discovery, share *config.Share) (dis case "k8s": return kubernetes.NewK8sDiscoveryRegister(share.RpcRegisterName.MessageGateway) case "etcd": - return getcd.NewSvcDiscoveryRegistry( + return etcd.NewSvcDiscoveryRegistry( discovery.Etcd.RootDirectory, discovery.Etcd.Address, - getcd.WithDialTimeout(10*time.Second), - getcd.WithMaxCallSendMsgSize(20*1024*1024), - getcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) + etcd.WithDialTimeout(10*time.Second), + etcd.WithMaxCallSendMsgSize(20*1024*1024), + etcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) case "direct": //return direct.NewConnDirect(config) default: From 6b463e9b028774230874bc53ca4ae133ff220afc Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 10:53:06 +0800 Subject: [PATCH 126/143] Add etcd as a service discovery mechanism --- pkg/common/discoveryregister/discoveryregister.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/common/discoveryregister/discoveryregister.go b/pkg/common/discoveryregister/discoveryregister.go index 1debc54e1..559c937c1 100644 --- a/pkg/common/discoveryregister/discoveryregister.go +++ b/pkg/common/discoveryregister/discoveryregister.go @@ -45,10 +45,7 @@ func NewDiscoveryRegister(discovery *config.Discovery, share *config.Share) (dis etcd.WithDialTimeout(10*time.Second), etcd.WithMaxCallSendMsgSize(20*1024*1024), etcd.WithUsernameAndPassword(discovery.Etcd.Username, discovery.Etcd.Password)) - case "direct": - //return direct.NewConnDirect(config) default: return nil, errs.New("unsupported discovery type", "type", discovery.Enable).Wrap() } - return nil, nil } From caf41c3e802211f18dadaceff8064d1ac6d7d992 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:06:44 +0800 Subject: [PATCH 127/143] Add etcd as a service discovery mechanism --- tools/check-component/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 5d4435e26..51daa0782 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -20,9 +20,9 @@ import ( "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister/etcd" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/discovery/etcd" "github.com/openimsdk/tools/discovery/zookeeper" "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/minio" From 5a6e774893b04fbd6729ebba0a63b95069ed17e1 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:13:29 +0800 Subject: [PATCH 128/143] Add etcd as a service discovery mechanism --- tools/check-component/main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 51daa0782..5fa84ac36 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -139,28 +139,28 @@ func main() { func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig *config.Redis, kafkaConfig *config.Kafka, minioConfig *config.Minio, discovery *config.Discovery, maxRetry int) error { checksDone := make(map[string]bool) - checks := map[string]func() error{ - "Mongo": func() error { + checks := map[string]func(ctx context.Context) error{ + "Mongo": func(ctx context.Context) error { return CheckMongo(ctx, mongoConfig) }, - "Redis": func() error { + "Redis": func(ctx context.Context) error { return CheckRedis(ctx, redisConfig) }, - "Kafka": func() error { + "Kafka": func(ctx context.Context) error { return CheckKafka(ctx, kafkaConfig) }, } if minioConfig != nil { - checks["MinIO"] = func() error { + checks["MinIO"] = func(ctx context.Context) error { return CheckMinIO(ctx, minioConfig) } } if discovery.Enable == "etcd" { - checks["Etcd"] = func() error { + checks["Etcd"] = func(ctx context.Context) error { return CheckEtcd(ctx, &discovery.Etcd) } } else if discovery.Enable == "zookeeper" { - checks["Zookeeper"] = func() error { + checks["Zookeeper"] = func(ctx context.Context) error { return CheckZookeeper(ctx, &discovery.ZooKeeper) } } @@ -169,7 +169,7 @@ func performChecks(ctx context.Context, mongoConfig *config.Mongo, redisConfig * allSuccess := true for name, check := range checks { if !checksDone[name] { - if err := check(); err != nil { + if err := check(ctx); err != nil { fmt.Printf("%s check failed: %v\n", name, err) allSuccess = false } else { From 9aae9d1de88257373aade7b92e3113767525845a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:47:52 +0800 Subject: [PATCH 129/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 640b4a824..7d20cc62b 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.10 + github.com/openimsdk/tools v0.0.49-alpha.11 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 241bb96aa..d010c6736 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.10 h1:vcfMUGBClD3TsjsTd/Wb0R2WcpjCo8hupMqreb0dXFA= -github.com/openimsdk/tools v0.0.49-alpha.10/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.11 h1:LybY7RVMwTqnBA5dLuZtI0SyBWyZj2oLqyzIieoNo5E= +github.com/openimsdk/tools v0.0.49-alpha.11/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 6e75b85f67f0965168a29395a4cecf70dd49b9ae Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:58:37 +0800 Subject: [PATCH 130/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7d20cc62b..841e014c3 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.11 + github.com/openimsdk/tools v0.0.49-alpha.12 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 From 62a640b77ec1b52b93adfff13ff21eb5e59a796d Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 11:59:10 +0800 Subject: [PATCH 131/143] Add etcd as a service discovery mechanism --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index d010c6736..d94d4c2f5 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.11 h1:LybY7RVMwTqnBA5dLuZtI0SyBWyZj2oLqyzIieoNo5E= -github.com/openimsdk/tools v0.0.49-alpha.11/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.12 h1:vsr63W1kHW1dEw9yelMhmr72WmsrjKfs2vXww3upfWI= +github.com/openimsdk/tools v0.0.49-alpha.12/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 26e4c0d3b69eb7b915372e5491e4faa7cf4e547e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 12:06:03 +0800 Subject: [PATCH 132/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 841e014c3..73ec1e59a 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.12 + github.com/openimsdk/tools v0.0.49-alpha.13 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index d94d4c2f5..4c331bc91 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.12 h1:vsr63W1kHW1dEw9yelMhmr72WmsrjKfs2vXww3upfWI= -github.com/openimsdk/tools v0.0.49-alpha.12/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.13 h1:u6r9eT90Co1gz/ukAjFAtxykU3JP4REWCgAy7WijVZ8= +github.com/openimsdk/tools v0.0.49-alpha.13/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 9543cd878dab28c77584595215ab48a3588485ff Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 12:15:22 +0800 Subject: [PATCH 133/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index a61399fb6..55b3ac481 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -87,12 +87,11 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M // Online push message for _, conn := range conns { conn := conn // loop var safe - ctx := ctx wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(conn) reply, err := msgClient.SuperGroupOnlineBatchPushOneMsg(ctx, input) if err != nil { - log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ", err, "req:", input.String()) + log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ", err, input.String()) return nil } From 404cf92ecc960627f1c9c19852a6046bbc6cfc21 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 12:18:09 +0800 Subject: [PATCH 134/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 55b3ac481..9a2c52f01 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -91,7 +91,7 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M msgClient := msggateway.NewMsgGatewayClient(conn) reply, err := msgClient.SuperGroupOnlineBatchPushOneMsg(ctx, input) if err != nil { - log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ", err, input.String()) + log.ZError(ctx, "SuperGroupOnlineBatchPushOneMsg ", err, "req:", input.String()) return nil } From c02fd2adc242de9eabe1400e07e5dc4613cd787a Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 12:26:50 +0800 Subject: [PATCH 135/143] Add etcd as a service discovery mechanism --- internal/push/onlinepusher.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index 9a2c52f01..a61399fb6 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -87,6 +87,7 @@ func (d *DefaultAllNode) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.M // Online push message for _, conn := range conns { conn := conn // loop var safe + ctx := ctx wg.Go(func() error { msgClient := msggateway.NewMsgGatewayClient(conn) reply, err := msgClient.SuperGroupOnlineBatchPushOneMsg(ctx, input) From 78af94fed55a383ed19c7946c4d4a719afe8e2bd Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 14:41:32 +0800 Subject: [PATCH 136/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 73ec1e59a..bc86e2efd 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.13 + github.com/openimsdk/tools v0.0.49-alpha.14 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 4c331bc91..8f2828e09 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.13 h1:u6r9eT90Co1gz/ukAjFAtxykU3JP4REWCgAy7WijVZ8= -github.com/openimsdk/tools v0.0.49-alpha.13/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.14 h1:1WigfCZhLQRYDdBjZRhf2ksFM4CWmMWd1y6RC36T02A= +github.com/openimsdk/tools v0.0.49-alpha.14/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From cff2461c5d151c8671513f474dc31ac37e68b365 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 14:49:14 +0800 Subject: [PATCH 137/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bc86e2efd..22efc0e1b 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.14 + github.com/openimsdk/tools v0.0.49-alpha.15 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 From 226ec40b3a9ce66d49321aaa1596b896e3ea8c0c Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 14:50:30 +0800 Subject: [PATCH 138/143] Add etcd as a service discovery mechanism --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 8f2828e09..a1e4832cd 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.14 h1:1WigfCZhLQRYDdBjZRhf2ksFM4CWmMWd1y6RC36T02A= -github.com/openimsdk/tools v0.0.49-alpha.14/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.15 h1:m9bvSLBnqLZ60pkeB2lvYOsprLnLphaFetoZCS75q0E= +github.com/openimsdk/tools v0.0.49-alpha.15/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 4973decaa55f9e95a9c74d1585da05c82e9508ca Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 15:23:10 +0800 Subject: [PATCH 139/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 22efc0e1b..9f48b93b0 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.15 + github.com/openimsdk/tools v0.0.49-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index a1e4832cd..acb199acd 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.15 h1:m9bvSLBnqLZ60pkeB2lvYOsprLnLphaFetoZCS75q0E= -github.com/openimsdk/tools v0.0.49-alpha.15/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.16 h1:AYL4DsE/k1uyfT5kcsdPbbloU5dSIGVERGstL0OgLwg= +github.com/openimsdk/tools v0.0.49-alpha.16/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 4cd92b33ee9449381240b951152dc0754d549ebf Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 15:30:10 +0800 Subject: [PATCH 140/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9f48b93b0..4a2d7a002 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.16 + github.com/openimsdk/tools v0.0.49-alpha.17 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index acb199acd..801783289 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.16 h1:AYL4DsE/k1uyfT5kcsdPbbloU5dSIGVERGstL0OgLwg= -github.com/openimsdk/tools v0.0.49-alpha.16/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.17 h1:cqPwj7AhVraRjAwLBk4BFKp9ASCbnh2GFFSRe9QgYCk= +github.com/openimsdk/tools v0.0.49-alpha.17/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From 55558b17f50d9487ad27212d0b7a9e1020d8e97b Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Tue, 14 May 2024 17:00:20 +0800 Subject: [PATCH 141/143] Add etcd as a service discovery mechanism --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4a2d7a002..e9777eaa8 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.17 + github.com/openimsdk/tools v0.0.49-alpha.18 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 801783289..3acb4709c 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.17 h1:cqPwj7AhVraRjAwLBk4BFKp9ASCbnh2GFFSRe9QgYCk= -github.com/openimsdk/tools v0.0.49-alpha.17/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.18 h1:ARQeCiRmExvtB6XYItegThuV63JGOTxddwhSLHYXd78= +github.com/openimsdk/tools v0.0.49-alpha.18/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= From dea7c0bf778ef42b0587d0d8eb6f1d673ad7150e Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Thu, 23 May 2024 11:02:26 +0800 Subject: [PATCH 142/143] etcd naming and discovery --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e9777eaa8..54e8a8e0e 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.65 - github.com/openimsdk/tools v0.0.49-alpha.18 + github.com/openimsdk/tools v0.0.49-alpha.19 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 From be0e1ee6c6982b02cd8040852f1ac4f22cf17aa0 Mon Sep 17 00:00:00 2001 From: skiffer-git <44203734@qq.com> Date: Thu, 23 May 2024 11:02:57 +0800 Subject: [PATCH 143/143] etcd naming and discovery --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 3acb4709c..5611a6ca6 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,8 @@ github.com/openimsdk/gomake v0.0.13 h1:xLDe/moqgWpRoptHzI4packAWzs4C16b+sVY+txNJ github.com/openimsdk/gomake v0.0.13/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.65 h1:SPT9qyUsFRTTKSKb/FjpS+xr6sxz/Kbnu+su1bxYagc= github.com/openimsdk/protocol v0.0.65/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.18 h1:ARQeCiRmExvtB6XYItegThuV63JGOTxddwhSLHYXd78= -github.com/openimsdk/tools v0.0.49-alpha.18/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= +github.com/openimsdk/tools v0.0.49-alpha.19 h1:CbASL0yefRSVAmWPVeRnhF7wZKd6umLfz31CIhEgrBs= +github.com/openimsdk/tools v0.0.49-alpha.19/go.mod h1:g7mkHXYUPi0/8aAX8VPMHpnb3hqdV69Jph+bXOGvvNM= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=