Merge branch 'openimsdk:main' into main

Dockerfile

@@ -3,16 +3,14 @@ FROM golang:1.20 AS builder
 
 # Set go mod installation source and proxy
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
+
 ENV GO111MODULE=$GO111MODULE
 ENV GOPROXY=$GOPROXY
 
 # Set up the working directory
 WORKDIR /openim/openim-server
 
-COPY go.mod go.sum ./
-RUN go mod download
-
 # Copy all files to the container
 ADD . .
 

build/images/openim-api/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-cmdutils/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-crontask/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-msggateway/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-msgtransfer/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-push/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-rpc-auth/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-rpc-conversation/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-rpc-friend/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-rpc-group/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-rpc-msg/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-rpc-third/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-rpc-user/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-tools/component/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

build/images/openim-tools/openim-web/Dockerfile

@@ -19,7 +19,7 @@
 FROM golang:1.20 AS builder
 
 ARG GO111MODULE=on
-ARG GOPROXY=https://goproxy.cn,direct
+ARG GOPROXY=https://goproxy.io,direct
 
 WORKDIR /openim/openim-server
 

config/templates/env.template

@@ -118,6 +118,7 @@ MONGO_OPENIM_PASSWORD=openIM123
 # Default: MONGO_DATABASE=openim_v3
 MONGO_DATABASE=openim_v3
 
+MONGO_MAX_POOL_SIZE=100
 # ----- Redis Configuration -----
 
 # Port on which Redis in-memory data structure store is running.

deployments/templates/config.yaml

@@ -535,3 +535,39 @@ prometheus:
   rtcPrometheusPort: [ ${RTC_PROM_PORT} ]
   thirdPrometheusPort: [ ${THIRD_PROM_PORT} ]
   messageTransferPrometheusPort: [ ${MSG_TRANSFER_PROM_PORT} ] # List of ports
+
+###################### LocalCache configuration information ######################
+# topic: redis subscriber channel
+# slotNum: number of slots, multiple slots can prevent too many keys from competing for a lock
+# slotSize: number of slots, the number of cached keys per slot, the overall cache quantity is slotNum * slotSize
+# successExpire: successful cache time seconds
+# failedExpire: failed cache time seconds
+# disable local caching and annotate topic, slotNum, and slotSize
+localCache:
+  user:
+    topic: DELETE_CACHE_USER
+    slotNum: 100
+    slotSize: 2000
+    successExpire: 300
+    failedExpire: 5
+
+  group:
+    topic: DELETE_CACHE_GROUP
+    slotNum: 100
+    slotSize: 2000
+    successExpire: 300
+    failedExpire: 5
+
+  friend:
+    topic: DELETE_CACHE_FRIEND
+    slotNum: 100
+    slotSize: 2000
+    successExpire: 300
+    failedExpire: 5
+
+  conversation:
+    topic: DELETE_CACHE_CONVERSATION
+    slotNum: 100
+    slotSize: 2000
+    successExpire: 300
+    failedExpire: 5

deployments/templates/env-template.yaml

@@ -118,6 +118,7 @@ MONGO_OPENIM_PASSWORD=${MONGO_OPENIM_PASSWORD}
 # Default: MONGO_DATABASE=openim_v3
 MONGO_DATABASE=${MONGO_DATABASE}
 
+MONGO_MAX_POOL_SIZE=${MONGO_MAX_POOL_SIZE}
 # ----- Redis Configuration -----
 
 # Port on which Redis in-memory data structure store is running.

docs/contrib/bash-log.md

@@ -2,6 +2,8 @@
 
 **PATH:** `scripts/lib/logging.sh`
 
+
+
 ### Introduction
 
 OpenIM, an intricate project, requires a robust logging mechanism to diagnose issues, maintain system health, and provide insights. A custom-built logging system embedded within OpenIM ensures consistent and structured logs. Let's delve into the design of this logging system and understand its various functions and their usage scenarios.

docs/contrib/go-code.md

@@ -266,13 +266,61 @@ errors.New("redis connection failed")
 - When specifying ranges of numbers, use inclusive ranges whenever possible.
 - Go 1.13 or above is recommended, and the error generation method is `fmt.Errorf("module xxx: %w", err)`.
 
-### 1.4 panic processing
+### 1.4 Panic Processing
+
+The use of `panic` should be carefully controlled in Go applications to ensure program stability and predictable error handling. Following are revised guidelines emphasizing the restriction on using `panic` and promoting alternative strategies for error handling and program termination.
+
+- **Prohibited in Business Logic:** Using `panic` within business logic processing is strictly prohibited. Business logic should handle errors gracefully and use error returns to propagate issues up the call stack.
+
+- **Restricted Use in Main Package:** In the main package, the use of `panic` should be reserved for situations where the program is entirely inoperable, such as failure to open essential files, inability to connect to the database, or other critical startup issues. Even in these scenarios, prefer using structured error handling to terminate the program.
+
+- **Use `log.Fatal` for Critical Errors:** Instead of panicking, use `log.Fatal` to log critical errors that prevent the program from operating normally. This approach allows the program to terminate while ensuring the error is properly logged for troubleshooting.
+
+- **Prohibition on Exportable Interfaces:** Exportable interfaces must not invoke `panic`. They should handle errors gracefully and return errors as part of their contract.
+
+- **Prefer Errors Over Panic:** It is recommended to use error returns instead of panic to convey errors within a package. This approach promotes error handling that integrates smoothly with Go's error handling idioms.
+
+#### Alternative to Panic: Structured Program Termination
+
+To enforce these guidelines, consider implementing structured functions to terminate the program gracefully in the face of unrecoverable errors, while providing clear error messages. Here are two recommended functions:
+
+```go
+// ExitWithError logs an error message and exits the program with a non-zero status.
+func ExitWithError(err error) {
+	progName := filepath.Base(os.Args[0])
+	fmt.Fprintf(os.Stderr, "%s exit -1: %+v\n", progName, err)
+	os.Exit(-1)
+}
+
+// SIGTERMExit logs a warning message when the program receives a SIGTERM signal and exits with status 0.
+func SIGTERMExit() {
+	progName := filepath.Base(os.Args[0])
+	fmt.Fprintf(os.Stderr, "Warning %s receive process terminal SIGTERM exit 0\n", progName)
+}
+```
+
+#### Example Usage:
+
+```go
+import (
+	_ "net/http/pprof"
+
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cmd"
+	util "github.com/openimsdk/open-im-server/v3/pkg/util/genutil"
+)
+
+func main() {
+	apiCmd := cmd.NewApiCmd()
+	apiCmd.AddPortFlag()
+	apiCmd.AddPrometheusPortFlag()
+	if err := apiCmd.Execute(); err != nil {
+		util.ExitWithError(err)
+	}
+}
+```
+
+In this example, `ExitWithError` is used to terminate the program when an unrecoverable error occurs, providing a clear error message to stderr and exiting with a non-zero status. This approach ensures that critical errors are logged and the program exits in a controlled manner, facilitating troubleshooting and maintaining the stability of the application.
 
-- Panic is prohibited in business logic processing.
-- In the main package, panic is only used when the program is completely inoperable, for example, the file cannot be opened, the database cannot be connected, and the program cannot run normally.
-- In the main package, use `log.Fatal` to record errors, so that the program can be terminated by the log, or the exception thrown by the panic can be recorded in the log file, which is convenient for troubleshooting.
-- An exportable interface must not panic.
-- It is recommended to use error instead of panic to convey errors in the package.
 
 ### 1.5 Unit Tests
 
@@ -439,10 +487,15 @@ var allowGitHook bool
 - Local variables should be as short as possible, for example, use buf to refer to buffer, and use i to refer to index.
 - The code automatically generated by the code generation tool can exclude this rule (such as the Id in `xxx.pb.go`)
 
-### 2.7 Constant naming
+### 2.7 Constant Naming
 
-- The constant name must follow the camel case, and the initial letter is uppercase or lowercase according to the access control decision.
-- If it is a constant of enumeration type, you need to create the corresponding type first:
+In Go, constants play a critical role in defining values that do not change throughout the execution of a program. Adhering to best practices in naming constants can significantly improve the readability and maintainability of your code. Here are some guidelines for constant naming:
+
+- **Camel Case Naming:** The name of a constant must follow the camel case notation. The initial letter should be uppercase or lowercase based on the access control requirements. Uppercase indicates that the constant is exported (visible outside the package), while lowercase indicates package-private visibility (visible only within its own package).
+
+- **Enumeration Type Constants:** For constants that represent a set of enumerated values, it's recommended to define a corresponding type first. This approach not only enhances type safety but also improves code readability by clearly indicating the purpose of the enumeration.
+
+**Example:**
 
 ```go
 // Code defines an error code type.
@@ -452,11 +505,40 @@ type Code int
 const (
      // ErrUnknown - 0: An unknown error occurred.
      ErrUnknown Code = iota
-     // ErrFatal - 1: An fatal error occurred.
+     // ErrFatal - 1: A fatal error occurred.
      ErrFatal
 )
 ```
 
+In the example above, `Code` is defined as a new type based on `int`. The enumerated constants `ErrUnknown` and `ErrFatal` are then defined with explicit comments to indicate their purpose and values. This pattern is particularly useful for grouping related constants and providing additional context.
+
+### Global Variables and Constants Across Packages
+
+- **Use Constants for Global Variables:** When defining variables that are intended to be accessed across packages, prefer using constants to ensure immutability. This practice avoids unintended modifications to the value, which can lead to unpredictable behavior or hard-to-track bugs.
+
+- **Lowercase for Package-Private Usage:** If a global variable or constant is intended for use only within its own package, it should start with a lowercase letter. This clearly signals its limited scope of visibility, adhering to Go's access control mechanism based on naming conventions.
+
+**Guideline:**
+
+- For global constants that need to be accessed across packages, declare them with an uppercase initial letter. This makes them exported, adhering to Go's visibility rules.
+- For constants used within the same package, start their names with a lowercase letter to limit their scope to the package.
+
+**Example:**
+
+```go
+package config
+
+// MaxConnections - the maximum number of allowed connections. Visible across packages.
+const MaxConnections int = 100
+
+// minIdleTime - the minimum idle time before a connection is considered stale. Only visible within the config package.
+const minIdleTime int = 30
+```
+
+In this example, `MaxConnections` is a global constant meant to be accessed across packages, hence it starts with an uppercase letter. On the other hand, `minIdleTime` is intended for use only within the `config` package, so it starts with a lowercase letter.
+
+Following these guidelines ensures that your Go code is more readable, maintainable, and consistent with Go's design philosophy and access control mechanisms.
+
 ### 2.8 Error naming
 
 - The Error type should be written in the form of FooError.
@@ -473,6 +555,190 @@ type ExitError struct {
 var ErrFormat = errors. New("unknown format")
 ```
 
+For non-standard Err naming, CICD will report an error
+
+
+### 2.9 Handling Errors Properly
+
+In Go, proper error handling is crucial for creating reliable and maintainable applications. It's important to ensure that errors are not ignored or discarded, as this can lead to unpredictable behavior and difficult-to-debug issues. Here are the guidelines and examples regarding the proper handling of errors.
+
+#### Guideline: Do Not Discard Errors
+
+- **Mandatory Error Propagation:** When calling a function that returns an error, the calling function must handle or propagate the error, instead of ignoring it. This approach ensures that errors are not silently ignored, allowing higher-level logic to make informed decisions about error handling.
+
+#### Incorrect Example: Discarding an Error
+
+```go
+package main
+
+import (
+	"io/ioutil"
+	"log"
+)
+
+func ReadFileContent(filename string) string {
+	content, _ := ioutil.ReadFile(filename) // Incorrect: Error is ignored
+	return string(content)
+}
+
+func main() {
+	content := ReadFileContent("example.txt")
+	log.Println(content)
+}
+```
+
+In this incorrect example, the error returned by `ioutil.ReadFile` is ignored. This can lead to situations where the program continues execution even if the file doesn't exist or cannot be accessed, potentially causing more cryptic errors downstream.
+
+#### Correct Example: Propagating an Error
+
+```go
+package main
+
+import (
+	"io/ioutil"
+	"log"
+)
+
+// ReadFileContent attempts to read and return the content of the specified file.
+// It returns an error if reading fails.
+func ReadFileContent(filename string) (string, error) {
+	content, err := ioutil.ReadFile(filename)
+	if err != nil {
+		// Correct: Propagate the error
+		return "", err
+	}
+	return string(content), nil
+}
+
+func main() {
+	content, err := ReadFileContent("example.txt")
+	if err != nil {
+		log.Fatalf("Failed to read file: %v", err)
+	}
+	log.Println(content)
+}
+```
+
+In the correct example, the error returned by `ioutil.ReadFile` is propagated back to the caller. The `main` function then checks the error and terminates the program with an appropriate error message if an error occurred. This approach ensures that errors are handled appropriately, and the program does not proceed with invalid state.
+
+### Best Practices for Error Handling
+
+1. **Always check the error returned by a function.** Do not ignore it.
+2. **Propagate errors up the call stack unless they can be handled gracefully at the current level.**
+3. **Provide context for errors when propagating them, making it easier to trace the source of the error.** This can be achieved using `fmt.Errorf` with the `%w` verb or dedicated wrapping functions provided by some error handling packages.
+4. **Log the error at the point where it is handled or makes the program to terminate, to provide insight into the failure.**
+
+By following these guidelines, you ensure that your Go applications handle errors in a consistent and effective manner, improving their reliability and maintainability.
+
+
+### 2.10 Using Context with IO or Inter-Process Communication (IPC)
+
+In Go, `context.Context` is a powerful construct for managing deadlines, cancellation signals, and other request-scoped values across API boundaries and between processes. It is particularly important in I/O operations or inter-process communication (IPC), where operations might need to be cancelled or timed out.
+
+#### Guideline: Use Context for IO and IPC
+
+- **Mandatory Use of Context:** When performing I/O operations or inter-process communication, it's crucial to use `context.Context` to manage the lifecycle of these operations. This includes setting deadlines, handling cancellation signals, and passing request-scoped values.
+
+#### Incorrect Example: Ignoring Context in an HTTP Call
+
+```go
+package main
+
+import (
+	"io/ioutil"
+	"net/http"
+	"log"
+)
+
+// FetchData makes an HTTP GET request to the specified URL and returns the response body.
+// This function does not use context, making it impossible to cancel the request or set a deadline.
+func FetchData(url string) (string, error) {
+	resp, err := http.Get(url) // Incorrect: Ignoring context
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return string(body), nil
+}
+
+func main() {
+	data, err := FetchData("http://example.com")
+	if err != nil {
+		log.Fatalf("Failed to fetch data: %v", err)
+	}
+	log.Println(data)
+}
+```
+
+In this incorrect example, the `FetchData` function makes an HTTP GET request without using a `context`. This approach does not allow the request to be cancelled or a timeout to be set, potentially leading to resources being wasted if the server takes too long to respond or if the operation needs to be aborted for any reason.
+
+#### Correct Example: Using Context in an HTTP Call
+
+```go
+package main
+
+import (
+	"context"
+	"io/ioutil"
+	"net/http"
+	"log"
+	"time"
+)
+
+// FetchDataWithContext makes an HTTP GET request to the specified URL using the provided context.
+// This allows the request to be cancelled or timed out according to the context's deadline.
+func FetchDataWithContext(ctx context.Context, url string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return string(body), nil
+}
+
+func main() {
+	// Create a context with a 5-second timeout
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	data, err := FetchDataWithContext(ctx, "http://example.com")
+	if err != nil {
+		log.Fatalf("Failed to fetch data: %v", err)
+	}
+	log.Println(data)
+}
+```
+
+In the correct example, `FetchDataWithContext` uses a context to make the HTTP GET request. This allows the operation to be cancelled or subjected to a timeout, as dictated by the context passed to it. The `context.WithTimeout` function is used in `main` to create a context that cancels the request if it takes longer than 5 seconds, demonstrating a practical use of context to manage operation lifecycle.
+
+### Best Practices for Using Context
+
+1. **Pass context as the first parameter of a function**, following the convention `func(ctx context.Context, ...)`.
+2. **Never ignore the context** provided to you in functions that support it. Always use it in your I/O or IPC operations.
+3. **Avoid storing context in a struct**. Contexts are meant to be passed around within the call stack, not stored.
+4. **Use context's cancellation and deadline features** to control the lifecycle of blocking operations, especially in network I/O and IPC scenarios.
+5. **Propagate context down the call stack** to any function that supports it, ensuring that your application can respond to cancellation signals and deadlines effectively.
+
+By adhering to these guidelines and examples, you can ensure that your Go applications handle I/O and IPC operations more reliably and efficiently, with proper support for cancellation, timeouts, and request-scoped values.
+
+
 ## 3. Comment specification
 
 - Each exportable name must have a comment, which briefly introduces the exported variables, functions, structures, interfaces, etc.

docs/contrib/logging.md

@@ -1,20 +1,386 @@
-##  Log Standards
+# OpenIM Logging and Error Handling Documentation
 
-### Log Standards
+## Script Logging Documentation Link
 
-- The unified log package `github.com/openimsdk/open-im-server/internal/pkg/log` should be used for all logging;
-- Use structured logging formats: `log.Infow`, `log.Warnw`, `log.Errorw`, etc. For example: `log.Infow("Update post function called")`;
-- All logs should start with an uppercase letter and should not end with a `.`. For example: `log.Infow("Update post function called")`;
-- Use past tense. For example, use `Could not delete B` instead of `Cannot delete B`;
-- Adhere to log level standards:
-  - Debug level logs use `log.Debugw`;
-  - Info level logs use `log.Infow`;
-  - Warning level logs use `log.Warnw`;
-  - Error level logs use `log.Errorw`;
-  - Panic level logs use `log.Panicw`;
-  - Fatal level logs use `log.Fatalw`.
-- Log settings:
-  - Development and test environments: The log level is set to `debug`, the log format can be set to `console` / `json` as needed, and caller is enabled;
-  - Production environment: The log level is set to `info`, the log format is set to `json`, and caller is enabled. (**Note**: In the early stages of going online, to facilitate troubleshooting, the log level can be set to `debug`)
-- When logging, avoid outputting sensitive information, such as passwords, keys, etc.
-- If you are calling a logging function in a function/method with a `context.Context` parameter, it is recommended to use `log.L(ctx).Infow()` for logging.
+If you wish to view the script's logging documentation, you can click on this link: [Logging Documentation](https://github.com/OpenIMSDK/Open-IM-Server/blob/main/docs/contrib/bash-log.md).
+
+Below is the documentation for logging and error handling in the OpenIM Go project.
+
+To create a standard set of documentation that is quick to read and easy to understand, we will highlight key information about the `Logger` interface and the `CodeError` interface. This includes the purpose of each interface, key methods, and their use cases. This will help developers quickly grasp how to effectively use logging and error handling within the project.
+
+## Logging (`Logger` Interface)
+
+### Purpose
+The `Logger` interface aims to provide the OpenIM project with a unified and flexible logging mechanism, supporting structured logging formats for efficient log management and analysis.
+
+### Key Methods
+
+- **Debug, Info, Warn, Error**  
+  Log messages of different levels to suit various logging needs and scenarios. These methods accept a context (`context.Context`), a message (`string`), and key-value pairs (`...interface{}`), allowing the log to carry rich context information.
+
+- **WithValues**  
+  Append key-value pair information to log messages, returning a new `Logger` instance. This helps in adding consistent context information.
+
+- **WithName**  
+  Set the name of the logger, which helps in identifying the source of the logs.
+
+- **WithCallDepth**  
+  Adjust the call stack depth to accurately identify the source of the log message.
+
+### Use Cases
+
+- Developers should choose the appropriate logging level (such as `Debug`, `Info`, `Warn`, `Error`) based on the importance of the information when logging.
+- Use `WithValues` and `WithName` to add richer context information to logs, facilitating subsequent tracking and analysis.
+
+## Error Handling (`CodeError` Interface)
+
+### Purpose
+The `CodeError` interface is designed to provide a unified mechanism for error handling and wrapping, making error information more detailed and manageable.
+
+### Key Methods
+
+- **Code**  
+  Return the error code to distinguish between different types of errors.
+
+- **Msg**  
+  Return the error message description to display to the user.
+
+- **Detail**  
+  Return detailed information about the error for further debugging by developers.
+
+- **WithDetail**  
+  Add detailed information to the error, returning a new `CodeError` instance.
+
+- **Is**  
+  Determine whether the current error matches a specified error, supporting a flexible error comparison mechanism.
+
+- **Wrap**  
+  Wrap another error with additional message description, facilitating the tracing of the error's cause.
+
+### Use Cases
+
+- When defining errors with specific codes and messages, use error types that implement the `CodeError` interface.
+- Use `WithDetail` to add additional context information to errors for more accurate problem localization.
+- Use the `Is` method to judge the type of error for conditional branching.
+- Use the `Wrap` method to wrap underlying errors while adding more contextual descriptions.
+
+## Logging Standards and Code Examples
+
+In the OpenIM project, we use the unified logging package `github.com/OpenIMSDK/tools/log` for logging to achieve efficient log management and analysis. This logging package supports structured logging formats, making it easier for developers to handle log information.
+
+### Logger Interface and Implementation
+
+The logger interface is defined as follows:
+
+```go
+type Logger interface {
+    Debug(ctx context.Context, msg string, keysAndValues ...interface{})
+    Info(ctx context.Context, msg string, keysAndValues ...interface{})
+    Warn(ctx context.Context, msg string, err error, keysAndValues ...interface{})
+    Error(ctx context.Context, msg string, err error, keysAndValues ...interface{})
+    WithValues(keysAndValues ...interface{}) Logger
+    WithName(name string) Logger
+    WithCallDepth(depth int) Logger
+}
+```
+
+Example code: Using the `Logger` interface to log at the info level.
+
+```go
+func main() {
+	logger := log.NewLogger().WithName("MyService")
+	ctx := context.Background()
+	logger.Info(ctx, "Service started", "port", "8080")
+}
+```
+
+## Error Handling and Code Examples
+
+We use the `github.com/OpenIMSDK/tools/errs` package for unified error handling and wrapping.
+
+### CodeError Interface and Implementation
+
+The error interface is defined as follows:
+
+```go
+type CodeError interface {
+    Code() int
+    Msg() string
+    Detail() string
+    WithDetail(detail string) CodeError
+    Is(err error, loose ...bool) bool
+    Wrap(msg ...string) error
+    error
+}
+```
+
+Example code: Creating and using the `CodeError` interface to handle errors.
+
+```go
+package main
+
+import (
+	"fmt"
+	"github.com/OpenIMSDK/tools/errs"
+)
+
+func main() {
+	err := errs.New(404, "Resource not found")
+	err = err.WithDetail("
+
+More details")
+	if e, ok := err.(errs.CodeError); ok {
+		fmt.Println(e.Code(), e.Msg(), e.Detail())
+	}
+}
+```
+
+### Detailed Logging Standards and Code Examples
+
+1. **Print key information at startup**  
+   It is crucial to print entry parameters and key process information at program startup. This helps understand the startup state and configuration of the program.
+
+   **Code Example**:
+   ```go
+   package main
+
+   import (
+       "fmt"
+       "os"
+   )
+
+   func main() {
+       fmt.Println("Program startup, version: 1.0.0")
+       fmt.Printf("Connecting to database: %s\n", os.Getenv("DATABASE_URL"))
+   }
+   ```
+
+2. **Use `tools/log` and `fmt` for logging**  
+   Logging should be done using a specialized logging library for unified management and formatted log output.
+
+   **Code Example**: Logging an info level message with `tools/log`.
+   ```go
+   package main
+
+   import (
+       "context"
+       "github.com/OpenIMSDK/tools/log"
+   )
+
+   func main() {
+       ctx := context.Background()
+       log.Info(ctx, "Application started successfully")
+   }
+   ```
+
+3. **Use standard error output for startup failures or critical information**  
+   Critical error messages or program startup failures should be indicated to the user through standard error output.
+
+   **Code Example**:
+   ```go
+   package main
+
+   import (
+       "fmt"
+       "os"
+   )
+
+   func checkEnvironment() bool {
+       return os.Getenv("REQUIRED_ENV") != ""
+   }
+
+   func main() {
+       if !checkEnvironment() {
+           fmt.Fprintln(os.Stderr, "Missing required environment variable")
+           os.Exit(1)
+       }
+   }
+   ```
+   
+   We encapsulate it into separate tools, which can output error information through the `tools/log` package.
+
+   ```go
+    package main
+
+    import (
+        util "github.com/openimsdk/open-im-server/v3/pkg/util/genutil"
+    )
+
+    func main() {
+        if err := apiCmd.Execute(); err != nil {
+            util.ExitWithError(err)
+        }
+    }
+    ```
+
+4. **Use `tools/log` package for runtime logging**  
+   This ensures consistency and control over logging.
+
+   **Code Example**: Same as the above example using `tools/log`. When `tools/log` is not initialized, consider using `fmt` for standard output.
+
+5. **Error logs should be printed by the top-level caller**  
+   This is to avoid duplicate logging of errors, typically errors are caught and logged at the application's outermost level.
+
+   **Code Example**:
+   ```go
+   package main
+
+   import (
+       "github.com/OpenIMSDK/tools/log"
+       "context"
+   )
+
+   func doSomething() error {
+       // An error occurs here
+       return errs.Wrap(errors.New("An error occurred"))
+   }
+
+   func controller() error {
+       err := doSomething()
+       if err != nil {
+           return err
+       }
+       return nil
+   }
+
+   func main() {
+       err := controller()
+       if err != nil {
+           log.Error(context.Background(), "Operation failed", err)
+       }
+   }
+   ```
+
+6. **Handling logs for API RPC calls and non-RPC applications**
+
+   For API RPC calls using gRPC, logs at the information level are printed by middleware on the gRPC server side, reducing the need to manually log in each RPC method. For non-RPC applications, it's recommended to manually log key execution paths to track the application's execution flow.
+
+    **gRPC Server-Side Logging Middleware:**
+
+    In gRPC, `UnaryInterceptor` and `StreamInterceptor` can intercept Unary and Stream type RPC calls, respectively. Here's an example of how to implement a simple Unary RPC logging middleware:
+
+    ```go
+    package main
+
+    import (
+        "context"
+        "google.golang.org/grpc"
+        "google.golang.org/grpc/codes"
+        "google.golang.org/grpc/status"
+        "log"
+        "time"
+    )
+
+    // unaryServerInterceptor returns a new unary server interceptor that logs each request.
+    func unaryServerInterceptor() grpc.UnaryServerInterceptor {
+        return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
+            // Record the start time of the request
+            start := time.Now()
+            // Call the actual RPC method
+            resp, err = handler(ctx, req)
+            // After the request ends, log the duration and other information
+            log.Printf("Request method: %s, duration: %s, error status: %v", info.FullMethod, time.Since(start), status.Code(err))
+            return resp, err
+        }
+    }
+
+    func main() {
+        // Create a gRPC server and add the middleware
+        s := grpc.NewServer
+
+(grpc.UnaryInterceptor(unaryServerInterceptor()))
+        // Register your service
+
+        // Start the gRPC server
+        log.Println("Starting gRPC server...")
+        // ...
+    }
+    ```
+
+    **Logging for Non-RPC Applications:**
+
+    For non-RPC applications, the key is to log at appropriate places in the code to maintain an execution trace. Here's a simple example showing how to log when handling a task:
+
+    ```go
+    package main
+
+    import (
+        "log"
+    )
+
+    func processTask(taskID string) {
+        // Log the start of task processing
+        log.Printf("Starting task processing: %s", taskID)
+        // Suppose this is where the task is processed
+
+        // Log after the task is completed
+        log.Printf("Task processing completed: %s", taskID)
+    }
+
+    func main() {
+        // Example task ID
+        taskID := "task123"
+        processTask(taskID)
+    }
+    ```
+
+    In both scenarios, appropriate logging can help developers and operators monitor the health of the system, trace the source of issues, and quickly locate and resolve problems. For gRPC logging, using middleware can effectively centralize log management and control. For non-RPC applications, ensuring logs are placed at critical execution points can help understand the program's operational flow and state changes.
+
+### When to Wrap Errors?
+
+1. **Wrap errors generated within the function**  
+   When an error occurs within a function, use `errs.Wrap` to add context information to the original error.
+
+   **Code Example**:
+   ```go
+   func doSomething() error {
+       // Suppose an error occurs here
+       err, _ := someFunc()
+       if err != nil {
+         return errs.Wrap(err, "doSomething failed")
+       }
+   }
+   ```
+
+2. **Wrap errors from system calls or other packages**  
+   When calling external libraries or system functions that return errors, also add context information to wrap the error.
+
+   **Code Example**:
+   ```go
+   func readConfig(file string) error {
+       _, err := os.ReadFile(file)
+       if err != nil {
+           return errs.Wrap(err, "Failed to read config file")
+       }
+       return nil
+   }
+   ```
+
+3. **No need to re-wrap errors for internal module calls**  
+
+   If an error has been appropriately wrapped with sufficient context information in an internal module call, there's no need to wrap it again.
+
+    **Code Example**:
+    ```go
+    func doSomething() error {
+        err := doAnotherThing()
+        if err != nil {
+            return err
+        }
+        return nil
+    }
+    ```
+
+4. **Ensure comprehensive wrapping of errors with detailed messages**
+   When wrapping errors, ensure to provide ample context information to make the error more understandable and easier to debug.
+
+   **Code Example**:
+   ```go
+   func connectDatabase() error {
+       err := db.Connect(config.DatabaseURL)
+       if err != nil {
+           return errs.Wrap(err, fmt.Sprintf("Failed to connect to database, URL: %s", config.DatabaseURL))
+       }
+       return nil
+   }
+   ```

go.mod

@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	firebase.google.com/go v3.13.0+incompatible
-	github.com/OpenIMSDK/protocol v0.0.55
+	github.com/OpenIMSDK/protocol v0.0.56
 	github.com/OpenIMSDK/tools v0.0.37
 	github.com/bwmarrin/snowflake v0.3.0 // indirect
 	github.com/dtm-labs/rockscache v0.1.1
@@ -18,6 +18,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.67
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
+	github.com/openimsdk/localcache v0.0.1
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.18.0
 	github.com/robfig/cron/v3 v3.0.1
@@ -84,6 +85,7 @@ require (
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
@@ -162,3 +164,5 @@ require (
 	golang.org/x/crypto v0.19.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )
+
+replace github.com/openimsdk/localcache => ./pkg/localcache

go.sum

@@ -18,8 +18,8 @@ firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIw
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/IBM/sarama v1.42.2 h1:VoY4hVIZ+WQJ8G9KNY/SQlWguBQXQ9uvFPOnrcu8hEw=
 github.com/IBM/sarama v1.42.2/go.mod h1:FLPGUGwYqEs62hq2bVG6Io2+5n+pS6s/WOXVKWSLFtE=
-github.com/OpenIMSDK/protocol v0.0.55 h1:eBjg8DyuhxGmuCUjpoZjg6MJJJXU/xJ3xJwFhrn34yA=
-github.com/OpenIMSDK/protocol v0.0.55/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
+github.com/OpenIMSDK/protocol v0.0.56 h1:mbVFyDBachEsmJLfYW5AU1z2KL8AUEpoHG8RPCIxjgg=
+github.com/OpenIMSDK/protocol v0.0.56/go.mod h1:F25dFrwrIx3lkNoiuf6FkCfxuwf8L4Z8UIsdTHP/r0Y=
 github.com/OpenIMSDK/tools v0.0.37 h1:qvDqmA4RbEJtPjZouWCkVuf/pjm6Y8nUrG5iH2gcnOg=
 github.com/OpenIMSDK/tools v0.0.37/go.mod h1:wBfR5CYmEyvxl03QJbTkhz1CluK6J4/lX0lviu8JAjE=
 github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
@@ -178,6 +178,8 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=

go.work

@@ -6,13 +6,11 @@ use (
     ./tools/codescan
 	./tools/changelog
 	./tools/component
-	./tools/data-conversion
 	./tools/formitychecker
 	./tools/imctl
 	./tools/infra
 	./tools/ncpu
 	./tools/openim-web
-	./tools/up35
 	./tools/url2im
 	./tools/versionchecker
 	./tools/yamlfmt

internal/api/route.go

@@ -50,7 +50,6 @@ import (
 )
 
 func Start(config *config.GlobalConfig, port int, proPort int) error {
-	log.ZDebug(context.Background(), "configAPI1111111111111111111", config, "port", port, "javafdasfs")
 	if port == 0 || proPort == 0 {
 		err := "port or proPort is empty:" + strconv.Itoa(port) + "," + strconv.Itoa(proPort)
 		return errs.Wrap(fmt.Errorf(err))

internal/msggateway/callback.go

@@ -100,7 +100,7 @@ func CallbackUserKickOff(ctx context.Context, globalConfig *config.GlobalConfig,
 // func callbackUserOnline(operationID, userID string, platformID int, token string, isAppBackground bool, connID
 // string) cbApi.CommonCallbackResp {
 //	callbackResp := cbApi.CommonCallbackResp{OperationID: operationID}
-//	if !config.Config.Callback.CallbackUserOnline.Enable {
+//	if !config.Config.Callback.CallbackUserOnline.WithEnable {
 //		return callbackResp
 //	}
 //	callbackUserOnlineReq := cbApi.CallbackUserOnlineReq{
@@ -129,7 +129,7 @@ func CallbackUserKickOff(ctx context.Context, globalConfig *config.GlobalConfig,
 //}
 //func callbackUserOffline(operationID, userID string, platformID int, connID string) cbApi.CommonCallbackResp {
 //	callbackResp := cbApi.CommonCallbackResp{OperationID: operationID}
-//	if !config.Config.Callback.CallbackUserOffline.Enable {
+//	if !config.Config.Callback.CallbackUserOffline.WithEnable {
 //		return callbackResp
 //	}
 //	callbackOfflineReq := cbApi.CallbackUserOfflineReq{
@@ -156,7 +156,7 @@ func CallbackUserKickOff(ctx context.Context, globalConfig *config.GlobalConfig,
 //}
 //func callbackUserKickOff(operationID string, userID string, platformID int) cbApi.CommonCallbackResp {
 //	callbackResp := cbApi.CommonCallbackResp{OperationID: operationID}
-//	if !config.Config.Callback.CallbackUserKickOff.Enable {
+//	if !config.Config.Callback.CallbackUserKickOff.WithEnable {
 //		return callbackResp
 //	}
 //	callbackUserKickOffReq := cbApi.CallbackUserKickOffReq{

internal/msggateway/hub_server.go

@@ -23,7 +23,6 @@ import (
 	"github.com/OpenIMSDK/tools/errs"
 	"github.com/OpenIMSDK/tools/log"
 	"github.com/OpenIMSDK/tools/mcontext"
-	"github.com/OpenIMSDK/tools/utils"
 	"github.com/openimsdk/open-im-server/v3/pkg/authverify"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
@@ -58,22 +57,25 @@ type Server struct {
 	rpcPort        int
 	prometheusPort int
 	LongConnServer LongConnServer
-	pushTerminal   []int
 	config         *config.GlobalConfig
+	pushTerminal   map[int]struct{}
 }
 
 func (s *Server) SetLongConnServer(LongConnServer LongConnServer) {
 	s.LongConnServer = LongConnServer
 }
 
-func NewServer(rpcPort int, proPort int, longConnServer LongConnServer, config *config.GlobalConfig) *Server {
-	return &Server{
+func NewServer(rpcPort int, proPort int, longConnServer LongConnServer, conf *config.GlobalConfig) *Server {
+	s := &Server{
 		rpcPort:        rpcPort,
 		prometheusPort: proPort,
 		LongConnServer: longConnServer,
-		pushTerminal:   []int{constant.IOSPlatformID, constant.AndroidPlatformID},
-		config:         config,
+		pushTerminal:   make(map[int]struct{}),
+		config:         conf,
 	}
+	s.pushTerminal[constant.IOSPlatformID] = struct{}{}
+	s.pushTerminal[constant.AndroidPlatformID] = struct{}{}
+	return s
 }
 
 func (s *Server) OnlinePushMsg(
@@ -127,13 +129,9 @@ func (s *Server) OnlineBatchPushOneMsg(
 	panic("implement me")
 }
 
-func (s *Server) SuperGroupOnlineBatchPushOneMsg(
-	ctx context.Context,
-	req *msggateway.OnlineBatchPushOneMsgReq,
+func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq,
 ) (*msggateway.OnlineBatchPushOneMsgResp, error) {
-
 	var singleUserResults []*msggateway.SingleMsgToUserResults
-
 	for _, v := range req.PushToUserIDs {
 		var resp []*msggateway.SingleMsgToUserPlatform
 		results := &msggateway.SingleMsgToUserResults{
@@ -154,23 +152,22 @@ func (s *Server) SuperGroupOnlineBatchPushOneMsg(
 			}
 
 			userPlatform := &msggateway.SingleMsgToUserPlatform{
-				RecvID:         v,
 				RecvPlatFormID: int32(client.PlatformID),
 			}
 			if !client.IsBackground ||
 				(client.IsBackground && client.PlatformID != constant.IOSPlatformID) {
 				err := client.PushMessage(ctx, req.MsgData)
 				if err != nil {
-					userPlatform.ResultCode = -2
+					userPlatform.ResultCode = int64(errs.ErrPushMsgErr.Code())
 					resp = append(resp, userPlatform)
 				} else {
-					if utils.IsContainInt(client.PlatformID, s.pushTerminal) {
+					if _, ok := s.pushTerminal[client.PlatformID]; ok {
 						results.OnlinePush = true
 						resp = append(resp, userPlatform)
 					}
 				}
 			} else {
-				userPlatform.ResultCode = -3
+				userPlatform.ResultCode = int64(errs.ErrIOSBackgroundPushErr.Code())
 				resp = append(resp, userPlatform)
 			}
 		}

internal/push/push_rpc_server.go

@@ -25,7 +25,7 @@ import (
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
-	"github.com/openimsdk/open-im-server/v3/pkg/common/db/localcache"
+	"github.com/openimsdk/open-im-server/v3/pkg/rpccache"
 	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
 	"google.golang.org/grpc"
 )
@@ -51,8 +51,8 @@ func Start(config *config.GlobalConfig, client discoveryregistry.SvcDiscoveryReg
 		client,
 		offlinePusher,
 		database,
-		localcache.NewGroupLocalCache(&groupRpcClient),
-		localcache.NewConversationLocalCache(&conversationRpcClient),
+		rpccache.NewGroupLocalCache(groupRpcClient, rdb),
+		rpccache.NewConversationLocalCache(conversationRpcClient, rdb),
 		&conversationRpcClient,
 		&groupRpcClient,
 		&msgRpcClient,

internal/push/push_to_client.go

@@ -36,9 +36,9 @@ import (
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
-	"github.com/openimsdk/open-im-server/v3/pkg/common/db/localcache"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics"
 	"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
+	"github.com/openimsdk/open-im-server/v3/pkg/rpccache"
 	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/grpc"
@@ -49,8 +49,8 @@ type Pusher struct {
 	database               controller.PushDatabase
 	discov                 discoveryregistry.SvcDiscoveryRegistry
 	offlinePusher          offlinepush.OfflinePusher
-	groupLocalCache        *localcache.GroupLocalCache
-	conversationLocalCache *localcache.ConversationLocalCache
+	groupLocalCache        *rpccache.GroupLocalCache
+	conversationLocalCache *rpccache.ConversationLocalCache
 	msgRpcClient           *rpcclient.MessageRpcClient
 	conversationRpcClient  *rpcclient.ConversationRpcClient
 	groupRpcClient         *rpcclient.GroupRpcClient
@@ -59,7 +59,7 @@ type Pusher struct {
 var errNoOfflinePusher = errors.New("no offlinePusher is configured")
 
 func NewPusher(config *config.GlobalConfig, discov discoveryregistry.SvcDiscoveryRegistry, offlinePusher offlinepush.OfflinePusher, database controller.PushDatabase,
-	groupLocalCache *localcache.GroupLocalCache, conversationLocalCache *localcache.ConversationLocalCache,
+	groupLocalCache *rpccache.GroupLocalCache, conversationLocalCache *rpccache.ConversationLocalCache,
 	conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient, msgRpcClient *rpcclient.MessageRpcClient,
 ) *Pusher {
 	return &Pusher{

internal/rpc/conversation/conversaion.go

@@ -48,14 +48,6 @@ type conversationServer struct {
 	config                         *config.GlobalConfig
 }
 
-func (c *conversationServer) GetConversationNotReceiveMessageUserIDs(
-	ctx context.Context,
-	req *pbconversation.GetConversationNotReceiveMessageUserIDsReq,
-) (*pbconversation.GetConversationNotReceiveMessageUserIDsResp, error) {
-	//TODO implement me
-	panic("implement me")
-}
-
 func Start(config *config.GlobalConfig, client discoveryregistry.SvcDiscoveryRegistry, server *grpc.Server) error {
 	rdb, err := cache.NewRedis(config)
 	if err != nil {
@@ -96,11 +88,11 @@ func (c *conversationServer) GetConversation(ctx context.Context, req *pbconvers
 	return resp, nil
 }
 
-func (m *conversationServer) GetSortedConversationList(ctx context.Context, req *pbconversation.GetSortedConversationListReq) (resp *pbconversation.GetSortedConversationListResp, err error) {
+func (c *conversationServer) GetSortedConversationList(ctx context.Context, req *pbconversation.GetSortedConversationListReq) (resp *pbconversation.GetSortedConversationListResp, err error) {
 	log.ZDebug(ctx, "GetSortedConversationList", "seqs", req, "userID", req.UserID)
 	var conversationIDs []string
 	if len(req.ConversationIDs) == 0 {
-		conversationIDs, err = m.conversationDatabase.GetConversationIDs(ctx, req.UserID)
+		conversationIDs, err = c.conversationDatabase.GetConversationIDs(ctx, req.UserID)
 		if err != nil {
 			return nil, err
 		}
@@ -108,7 +100,7 @@ func (m *conversationServer) GetSortedConversationList(ctx context.Context, req
 		conversationIDs = req.ConversationIDs
 	}
 
-	conversations, err := m.conversationDatabase.FindConversations(ctx, req.UserID, conversationIDs)
+	conversations, err := c.conversationDatabase.FindConversations(ctx, req.UserID, conversationIDs)
 	if err != nil {
 		return nil, err
 	}
@@ -116,22 +108,22 @@ func (m *conversationServer) GetSortedConversationList(ctx context.Context, req
 		return nil, errs.ErrRecordNotFound.Wrap()
 	}
 
-	maxSeqs, err := m.msgRpcClient.GetMaxSeqs(ctx, conversationIDs)
+	maxSeqs, err := c.msgRpcClient.GetMaxSeqs(ctx, conversationIDs)
 	if err != nil {
 		return nil, err
 	}
 
-	chatLogs, err := m.msgRpcClient.GetMsgByConversationIDs(ctx, conversationIDs, maxSeqs)
+	chatLogs, err := c.msgRpcClient.GetMsgByConversationIDs(ctx, conversationIDs, maxSeqs)
 	if err != nil {
 		return nil, err
 	}
 
-	conversationMsg, err := m.getConversationInfo(ctx, chatLogs, req.UserID)
+	conversationMsg, err := c.getConversationInfo(ctx, chatLogs, req.UserID)
 	if err != nil {
 		return nil, err
 	}
 
-	hasReadSeqs, err := m.msgRpcClient.GetHasReadSeqs(ctx, req.UserID, conversationIDs)
+	hasReadSeqs, err := c.msgRpcClient.GetHasReadSeqs(ctx, req.UserID, conversationIDs)
 	if err != nil {
 		return nil, err
 	}
@@ -163,8 +155,8 @@ func (m *conversationServer) GetSortedConversationList(ctx context.Context, req
 		UnreadTotal:       unreadTotal,
 	}
 
-	m.conversationSort(conversation_isPinTime, resp, conversation_unreadCount, conversationMsg)
-	m.conversationSort(conversation_notPinTime, resp, conversation_unreadCount, conversationMsg)
+	c.conversationSort(conversation_isPinTime, resp, conversation_unreadCount, conversationMsg)
+	c.conversationSort(conversation_notPinTime, resp, conversation_unreadCount, conversationMsg)
 
 	resp.ConversationElems = utils.Paginate(resp.ConversationElems, int(req.Pagination.GetPageNumber()), int(req.Pagination.GetShowNumber()))
 	return resp, nil
@@ -535,3 +527,14 @@ func (c *conversationServer) getConversationInfo(
 	}
 	return conversationMsg, nil
 }
+
+func (c *conversationServer) GetConversationNotReceiveMessageUserIDs(
+	ctx context.Context,
+	req *pbconversation.GetConversationNotReceiveMessageUserIDsReq,
+) (*pbconversation.GetConversationNotReceiveMessageUserIDsResp, error) {
+	userIDs, err := c.conversationDatabase.GetConversationNotReceiveMessageUserIDs(ctx, req.ConversationID)
+	if err != nil {
+		return nil, err
+	}
+	return &pbconversation.GetConversationNotReceiveMessageUserIDsResp{UserIDs: userIDs}, nil
+}

internal/rpc/group/group.go

@@ -1068,6 +1068,7 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq)
 		group []*relationtb.GroupModel
 		err   error
 	)
+
 	if req.GroupID != "" {
 		group, err = s.db.FindGroup(ctx, []string{req.GroupID})
 		resp.Total = uint32(len(group))
@@ -1080,15 +1081,7 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq)
 		return nil, err
 	}
 
-	var groups []*relationtb.GroupModel
-	for _, v := range group {
-		if v.Status == constant.GroupStatusDismissed {
-			resp.Total--
-			continue
-		}
-		groups = append(groups, v)
-	}
-	groupIDs := utils.Slice(groups, func(e *relationtb.GroupModel) string {
+	groupIDs := utils.Slice(group, func(e *relationtb.GroupModel) string {
 		return e.GroupID
 	})
 	ownerMembers, err := s.db.FindGroupsOwner(ctx, groupIDs)
@@ -1102,7 +1095,7 @@ func (s *groupServer) GetGroups(ctx context.Context, req *pbgroup.GetGroupsReq)
 	if err != nil {
 		return nil, err
 	}
-	resp.Groups = utils.Slice(groups, func(group *relationtb.GroupModel) *pbgroup.CMSGroup {
+	resp.Groups = utils.Slice(group, func(group *relationtb.GroupModel) *pbgroup.CMSGroup {
 		var (
 			userID   string
 			username string

internal/rpc/msg/as_read.go

@@ -41,7 +41,7 @@ func (m *msgServer) GetConversationsHasReadAndMaxSeq(ctx context.Context, req *m
 	if err != nil {
 		return nil, err
 	}
-	conversations, err := m.Conversation.GetConversations(ctx, req.UserID, conversationIDs)
+	conversations, err := m.ConversationLocalCache.GetConversations(ctx, req.UserID, conversationIDs)
 	if err != nil {
 		return nil, err
 	}
@@ -104,7 +104,7 @@ func (m *msgServer) MarkMsgsAsRead(
 	if hasReadSeq > maxSeq {
 		return nil, errs.ErrArgs.Wrap("hasReadSeq must not be bigger than maxSeq")
 	}
-	conversation, err := m.Conversation.GetConversation(ctx, req.UserID, req.ConversationID)
+	conversation, err := m.ConversationLocalCache.GetConversation(ctx, req.UserID, req.ConversationID)
 	if err != nil {
 		return
 	}
@@ -144,7 +144,7 @@ func (m *msgServer) MarkConversationAsRead(
 	ctx context.Context,
 	req *msg.MarkConversationAsReadReq,
 ) (resp *msg.MarkConversationAsReadResp, err error) {
-	conversation, err := m.Conversation.GetConversation(ctx, req.UserID, req.ConversationID)
+	conversation, err := m.ConversationLocalCache.GetConversation(ctx, req.UserID, req.ConversationID)
 	if err != nil {
 		return nil, err
 	}

internal/rpc/msg/revoke.go

@@ -44,7 +44,7 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg.
 	if err := authverify.CheckAccessV3(ctx, req.UserID, m.config); err != nil {
 		return nil, err
 	}
-	user, err := m.User.GetUserInfo(ctx, req.UserID)
+	user, err := m.UserLocalCache.GetUserInfo(ctx, req.UserID)
 	if err != nil {
 		return nil, err
 	}
@@ -70,12 +70,7 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg.
 			}
 			role = user.AppMangerLevel
 		case constant.SuperGroupChatType:
-			members, err := m.Group.GetGroupMemberInfoMap(
-				ctx,
-				msgs[0].GroupID,
-				utils.Distinct([]string{req.UserID, msgs[0].SendID}),
-				true,
-			)
+			members, err := m.GroupLocalCache.GetGroupMemberInfoMap(ctx, msgs[0].GroupID, utils.Distinct([]string{req.UserID, msgs[0].SendID}))
 			if err != nil {
 				return nil, err
 			}

internal/rpc/msg/send.go

@@ -97,7 +97,7 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa
 	}
 	tagAll := utils.IsContain(constant.AtAllString, msg.AtUserIDList)
 	if tagAll {
-		memberUserIDList, err := m.Group.GetGroupMemberIDs(ctx, msg.GroupID)
+		memberUserIDList, err := m.GroupLocalCache.GetGroupMemberIDs(ctx, msg.GroupID)
 		if err != nil {
 			log.ZWarn(ctx, "GetGroupMemberIDs", err)
 			return
@@ -143,6 +143,7 @@ func (m *msgServer) sendMsgNotification(
 }
 
 func (m *msgServer) sendMsgSingleChat(ctx context.Context, req *pbmsg.SendMsgReq) (resp *pbmsg.SendMsgResp, err error) {
+	log.ZDebug(ctx, "sendMsgSingleChat return")
 	if err := m.messageVerification(ctx, req); err != nil {
 		return nil, err
 	}

internal/rpc/msg/server.go

@@ -22,8 +22,8 @@ import (
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/cache"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/controller"
-	"github.com/openimsdk/open-im-server/v3/pkg/common/db/localcache"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/db/unrelation"
+	"github.com/openimsdk/open-im-server/v3/pkg/rpccache"
 	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
 	"google.golang.org/grpc"
 )
@@ -33,12 +33,11 @@ type (
 	msgServer               struct {
 		RegisterCenter         discoveryregistry.SvcDiscoveryRegistry
 		MsgDatabase            controller.CommonMsgDatabase
-		Group                  *rpcclient.GroupRpcClient
-		User                   *rpcclient.UserRpcClient
 		Conversation           *rpcclient.ConversationRpcClient
-		friend                 *rpcclient.FriendRpcClient
-		GroupLocalCache        *localcache.GroupLocalCache
-		ConversationLocalCache *localcache.ConversationLocalCache
+		UserLocalCache         *rpccache.UserLocalCache
+		FriendLocalCache       *rpccache.FriendLocalCache
+		GroupLocalCache        *rpccache.GroupLocalCache
+		ConversationLocalCache *rpccache.ConversationLocalCache
 		Handlers               MessageInterceptorChain
 		notificationSender     *rpcclient.NotificationSender
 		config                 *config.GlobalConfig
@@ -84,13 +83,12 @@ func Start(config *config.GlobalConfig, client discoveryregistry.SvcDiscoveryReg
 	}
 	s := &msgServer{
 		Conversation:           &conversationClient,
-		User:                   &userRpcClient,
-		Group:                  &groupRpcClient,
 		MsgDatabase:            msgDatabase,
 		RegisterCenter:         client,
-		GroupLocalCache:        localcache.NewGroupLocalCache(&groupRpcClient),
-		ConversationLocalCache: localcache.NewConversationLocalCache(&conversationClient),
-		friend:                 &friendRpcClient,
+		UserLocalCache:         rpccache.NewUserLocalCache(userRpcClient, rdb),
+		GroupLocalCache:        rpccache.NewGroupLocalCache(groupRpcClient, rdb),
+		ConversationLocalCache: rpccache.NewConversationLocalCache(conversationClient, rdb),
+		FriendLocalCache:       rpccache.NewFriendLocalCache(friendRpcClient, rdb),
 		config:                 config,
 	}
 	s.notificationSender = rpcclient.NewNotificationSender(config, rpcclient.WithLocalSendMsg(s.SendMsg))

internal/rpc/msg/statistics.go

@@ -40,7 +40,7 @@ func (m *msgServer) GetActiveUser(ctx context.Context, req *msg.GetActiveUserReq
 	var pbUsers []*msg.ActiveUser
 	if len(users) > 0 {
 		userIDs := utils.Slice(users, func(e *unrelation.UserCount) string { return e.UserID })
-		userMap, err := m.User.GetUsersInfoMap(ctx, userIDs)
+		userMap, err := m.UserLocalCache.GetUsersInfoMap(ctx, userIDs)
 		if err != nil {
 			return nil, err
 		}
@@ -82,7 +82,7 @@ func (m *msgServer) GetActiveGroup(ctx context.Context, req *msg.GetActiveGroupR
 	var pbgroups []*msg.ActiveGroup
 	if len(groups) > 0 {
 		groupIDs := utils.Slice(groups, func(e *unrelation.GroupCount) string { return e.GroupID })
-		resp, err := m.Group.GetGroupInfos(ctx, groupIDs, false)
+		resp, err := m.GroupLocalCache.GetGroupInfos(ctx, groupIDs)
 		if err != nil {
 			return nil, err
 		}

internal/rpc/msg/sync_msg.go

@@ -35,7 +35,7 @@ func (m *msgServer) PullMessageBySeqs(
 	resp.NotificationMsgs = make(map[string]*sdkws.PullMsgs)
 	for _, seq := range req.SeqRanges {
 		if !msgprocessor.IsNotification(seq.ConversationID) {
-			conversation, err := m.Conversation.GetConversation(ctx, req.UserID, seq.ConversationID)
+			conversation, err := m.ConversationLocalCache.GetConversation(ctx, req.UserID, seq.ConversationID)
 			if err != nil {
 				log.ZError(ctx, "GetConversation error", err, "conversationID", seq.ConversationID)
 				continue
@@ -131,14 +131,14 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq
 			sendIDs = append(sendIDs, chatLog.SendID)
 		}
 		switch chatLog.SessionType {
-		case constant.SingleChatType:
+		case constant.SingleChatType, constant.NotificationChatType:
 			recvIDs = append(recvIDs, chatLog.RecvID)
 		case constant.GroupChatType, constant.SuperGroupChatType:
 			groupIDs = append(groupIDs, chatLog.GroupID)
 		}
 	}
 	if len(sendIDs) != 0 {
-		sendInfos, err := m.User.GetUsersInfo(ctx, sendIDs)
+		sendInfos, err := m.UserLocalCache.GetUsersInfo(ctx, sendIDs)
 		if err != nil {
 			return nil, err
 		}
@@ -147,7 +147,7 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq
 		}
 	}
 	if len(recvIDs) != 0 {
-		recvInfos, err := m.User.GetUsersInfo(ctx, recvIDs)
+		recvInfos, err := m.UserLocalCache.GetUsersInfo(ctx, recvIDs)
 		if err != nil {
 			return nil, err
 		}
@@ -156,7 +156,7 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq
 		}
 	}
 	if len(groupIDs) != 0 {
-		groupInfos, err := m.Group.GetGroupInfos(ctx, groupIDs, true)
+		groupInfos, err := m.GroupLocalCache.GetGroupInfos(ctx, groupIDs)
 		if err != nil {
 			return nil, err
 		}
@@ -173,7 +173,7 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq
 			pbchatLog.SenderNickname = sendMap[chatLog.SendID]
 		}
 		switch chatLog.SessionType {
-		case constant.SingleChatType:
+		case constant.SingleChatType, constant.NotificationChatType:
 			pbchatLog.RecvNickname = recvMap[chatLog.RecvID]
 
 		case constant.GroupChatType, constant.SuperGroupChatType:

internal/rpc/msg/verify.go

@@ -24,6 +24,7 @@ import (
 	"github.com/OpenIMSDK/protocol/msg"
 	"github.com/OpenIMSDK/protocol/sdkws"
 	"github.com/OpenIMSDK/tools/errs"
+	"github.com/OpenIMSDK/tools/log"
 	"github.com/OpenIMSDK/tools/utils"
 )
 
@@ -59,15 +60,15 @@ func (m *msgServer) messageVerification(ctx context.Context, data *msg.SendMsgRe
 			data.MsgData.ContentType >= constant.NotificationBegin {
 			return nil
 		}
-		black, err := m.friend.IsBlocked(ctx, data.MsgData.SendID, data.MsgData.RecvID)
+		black, err := m.FriendLocalCache.IsBlack(ctx, data.MsgData.SendID, data.MsgData.RecvID)
 		if err != nil {
 			return err
 		}
 		if black {
 			return errs.ErrBlockedByPeer.Wrap()
 		}
-		if *m.config.MessageVerify.FriendVerify {
-			friend, err := m.friend.IsFriend(ctx, data.MsgData.SendID, data.MsgData.RecvID)
+		if m.config.MessageVerify.FriendVerify != nil && *m.config.MessageVerify.FriendVerify {
+			friend, err := m.FriendLocalCache.IsFriend(ctx, data.MsgData.SendID, data.MsgData.RecvID)
 			if err != nil {
 				return err
 			}
@@ -78,7 +79,7 @@ func (m *msgServer) messageVerification(ctx context.Context, data *msg.SendMsgRe
 		}
 		return nil
 	case constant.SuperGroupChatType:
-		groupInfo, err := m.Group.GetGroupInfoCache(ctx, data.MsgData.GroupID)
+		groupInfo, err := m.GroupLocalCache.GetGroupInfo(ctx, data.MsgData.GroupID)
 		if err != nil {
 			return err
 		}
@@ -99,17 +100,17 @@ func (m *msgServer) messageVerification(ctx context.Context, data *msg.SendMsgRe
 			data.MsgData.ContentType >= constant.NotificationBegin {
 			return nil
 		}
-		// memberIDs, err := m.GroupLocalCache.GetGroupMemberIDs(ctx, data.MsgData.GroupID)
-		// if err != nil {
-		// 	return err
-		// }
-		// if !utils.IsContain(data.MsgData.SendID, memberIDs) {
-		// 	return errs.ErrNotInGroupYet.Wrap()
-		// }
-
-		groupMemberInfo, err := m.Group.GetGroupMemberCache(ctx, data.MsgData.GroupID, data.MsgData.SendID)
+		memberIDs, err := m.GroupLocalCache.GetGroupMemberIDMap(ctx, data.MsgData.GroupID)
 		if err != nil {
-			if err == errs.ErrRecordNotFound {
+			return err
+		}
+		if _, ok := memberIDs[data.MsgData.SendID]; !ok {
+			return errs.ErrNotInGroupYet.Wrap()
+		}
+
+		groupMemberInfo, err := m.GroupLocalCache.GetGroupMember(ctx, data.MsgData.GroupID, data.MsgData.SendID)
+		if err != nil {
+			if errs.ErrRecordNotFound.Is(err) {
 				return errs.ErrNotInGroupYet.Wrap(err.Error())
 			}
 			return err
@@ -157,6 +158,9 @@ func (m *msgServer) encapsulateMsgData(msg *sdkws.MsgData) {
 	case constant.Custom:
 		fallthrough
 	case constant.Quote:
+		utils.SetSwitchFromOptions(msg.Options, constant.IsConversationUpdate, true)
+		utils.SetSwitchFromOptions(msg.Options, constant.IsUnreadCount, true)
+		utils.SetSwitchFromOptions(msg.Options, constant.IsSenderSync, true)
 	case constant.Revoke:
 		utils.SetSwitchFromOptions(msg.Options, constant.IsUnreadCount, false)
 		utils.SetSwitchFromOptions(msg.Options, constant.IsOfflinePush, false)
@@ -187,7 +191,8 @@ func (m *msgServer) modifyMessageByUserMessageReceiveOpt(
 	sessionType int,
 	pb *msg.SendMsgReq,
 ) (bool, error) {
-	opt, err := m.User.GetUserGlobalMsgRecvOpt(ctx, userID)
+	defer log.ZDebug(ctx, "modifyMessageByUserMessageReceiveOpt return")
+	opt, err := m.UserLocalCache.GetUserGlobalMsgRecvOpt(ctx, userID)
 	if err != nil {
 		return false, err
 	}
@@ -203,7 +208,7 @@ func (m *msgServer) modifyMessageByUserMessageReceiveOpt(
 		return true, nil
 	}
 	// conversationID := utils.GetConversationIDBySessionType(conversationID, sessionType)
-	singleOpt, err := m.Conversation.GetSingleConversationRecvMsgOpt(ctx, userID, conversationID)
+	singleOpt, err := m.ConversationLocalCache.GetSingleConversationRecvMsgOpt(ctx, userID, conversationID)
 	if errs.ErrRecordNotFound.Is(err) {
 		return true, nil
 	} else if err != nil {

internal/rpc/third/log.go

@@ -133,6 +133,13 @@ func (t *thirdServer) SearchLogs(ctx context.Context, req *third.SearchLogsReq)
 	if req.StartTime > req.EndTime {
 		return nil, errs.ErrArgs.Wrap("startTime>endTime")
 	}
+	if req.StartTime == 0 && req.EndTime == 0 {
+		t := time.Date(2019, time.January, 1, 0, 0, 0, 0, time.UTC)
+		timestampMills := t.UnixNano() / int64(time.Millisecond)
+		req.StartTime = timestampMills
+		req.EndTime = time.Now().UnixNano() / int64(time.Millisecond)
+	}
+
 	total, logs, err := t.thirdDatabase.SearchLogs(ctx, req.Keyword, time.UnixMilli(req.StartTime), time.UnixMilli(req.EndTime), req.Pagination)
 	if err != nil {
 		return nil, err

pkg/common/cachekey/black.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 OpenIM. All rights reserved.
+// Copyright © 2024 OpenIM. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,23 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package cachekey
 
-import (
-	"flag"
-	"log"
-	"os"
-
-	"github.com/openimsdk/open-im-server/v3/tools/up35/pkg"
+const (
+	BlackIDsKey = "BLACK_IDS:"
+	IsBlackKey  = "IS_BLACK:" // local cache
 )
 
-func main() {
-	var path string
-	flag.StringVar(&path, "c", "", "path config file")
-	flag.Parse()
-	if err := pkg.Main(path); err != nil {
-		log.Fatal(err)
-		return
-	}
-	os.Exit(0)
+func GetBlackIDsKey(ownerUserID string) string {
+	return BlackIDsKey + ownerUserID
+
+}
+
+func GetIsBlackIDsKey(possibleBlackUserID, userID string) string {
+	return IsBlackKey + userID + "-" + possibleBlackUserID
 }

pkg/common/cachekey/conversation.go

@@ -0,0 +1,58 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cachekey
+
+const (
+	ConversationKey                          = "CONVERSATION:"
+	ConversationIDsKey                       = "CONVERSATION_IDS:"
+	ConversationIDsHashKey                   = "CONVERSATION_IDS_HASH:"
+	ConversationHasReadSeqKey                = "CONVERSATION_HAS_READ_SEQ:"
+	RecvMsgOptKey                            = "RECV_MSG_OPT:"
+	SuperGroupRecvMsgNotNotifyUserIDsKey     = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
+	SuperGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
+	ConversationNotReceiveMessageUserIDsKey  = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:"
+)
+
+func GetConversationKey(ownerUserID, conversationID string) string {
+	return ConversationKey + ownerUserID + ":" + conversationID
+}
+
+func GetConversationIDsKey(ownerUserID string) string {
+	return ConversationIDsKey + ownerUserID
+}
+
+func GetSuperGroupRecvNotNotifyUserIDsKey(groupID string) string {
+	return SuperGroupRecvMsgNotNotifyUserIDsKey + groupID
+}
+
+func GetRecvMsgOptKey(ownerUserID, conversationID string) string {
+	return RecvMsgOptKey + ownerUserID + ":" + conversationID
+}
+
+func GetSuperGroupRecvNotNotifyUserIDsHashKey(groupID string) string {
+	return SuperGroupRecvMsgNotNotifyUserIDsHashKey + groupID
+}
+
+func GetConversationHasReadSeqKey(ownerUserID, conversationID string) string {
+	return ConversationHasReadSeqKey + ownerUserID + ":" + conversationID
+}
+
+func GetConversationNotReceiveMessageUserIDsKey(conversationID string) string {
+	return ConversationNotReceiveMessageUserIDsKey + conversationID
+}
+
+func GetUserConversationIDsHashKey(ownerUserID string) string {
+	return ConversationIDsHashKey + ownerUserID
+}

pkg/common/cachekey/friend.go

@@ -0,0 +1,38 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cachekey
+
+const (
+	FriendIDsKey        = "FRIEND_IDS:"
+	TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:"
+	FriendKey           = "FRIEND_INFO:"
+	IsFriendKey         = "IS_FRIEND:" // local cache key
+)
+
+func GetFriendIDsKey(ownerUserID string) string {
+	return FriendIDsKey + ownerUserID
+}
+
+func GetTwoWayFriendsIDsKey(ownerUserID string) string {
+	return TwoWayFriendsIDsKey + ownerUserID
+}
+
+func GetFriendKey(ownerUserID, friendUserID string) string {
+	return FriendKey + ownerUserID + "-" + friendUserID
+}
+
+func GetIsFriendKey(possibleFriendUserID, userID string) string {
+	return IsFriendKey + possibleFriendUserID + "-" + userID
+}

pkg/common/cachekey/group.go

@@ -0,0 +1,59 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cachekey
+
+import (
+	"strconv"
+	"time"
+)
+
+const (
+	groupExpireTime            = time.Second * 60 * 60 * 12
+	GroupInfoKey               = "GROUP_INFO:"
+	GroupMemberIDsKey          = "GROUP_MEMBER_IDS:"
+	GroupMembersHashKey        = "GROUP_MEMBERS_HASH2:"
+	GroupMemberInfoKey         = "GROUP_MEMBER_INFO:"
+	JoinedGroupsKey            = "JOIN_GROUPS_KEY:"
+	GroupMemberNumKey          = "GROUP_MEMBER_NUM_CACHE:"
+	GroupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:"
+)
+
+func GetGroupInfoKey(groupID string) string {
+	return GroupInfoKey + groupID
+}
+
+func GetJoinedGroupsKey(userID string) string {
+	return JoinedGroupsKey + userID
+}
+
+func GetGroupMembersHashKey(groupID string) string {
+	return GroupMembersHashKey + groupID
+}
+
+func GetGroupMemberIDsKey(groupID string) string {
+	return GroupMemberIDsKey + groupID
+}
+
+func GetGroupMemberInfoKey(groupID, userID string) string {
+	return GroupMemberInfoKey + groupID + "-" + userID
+}
+
+func GetGroupMemberNumKey(groupID string) string {
+	return GroupMemberNumKey + groupID
+}
+
+func GetGroupRoleLevelMemberIDsKey(groupID string, roleLevel int32) string {
+	return GroupRoleLevelMemberIDsKey + groupID + "-" + strconv.Itoa(int(roleLevel))
+}

pkg/common/cachekey/user.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 OpenIM. All rights reserved.
+// Copyright © 2024 OpenIM. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package common
+package cachekey
 
-import "fmt"
+const (
+	UserInfoKey             = "USER_INFO:"
+	UserGlobalRecvMsgOptKey = "USER_GLOBAL_RECV_MSG_OPT_KEY:"
+)
 
-func ErrorPrint(s string) {
-	fmt.Printf("\x1b[%dm%v\x1b[0m\n", 31, s)
+func GetUserInfoKey(userID string) string {
+	return UserInfoKey + userID
 }
 
-func SuccessPrint(s string) {
-	fmt.Printf("\x1b[%dm%v\x1b[0m\n", 32, s)
-}
-
-func WarningPrint(s string) {
-	fmt.Printf("\x1b[%dmWarning: But %v\x1b[0m\n", 33, s)
+func GetUserGlobalRecvMsgOptKey(userID string) string {
+	return UserGlobalRecvMsgOptKey + userID
 }

pkg/common/config/config.go

@@ -16,6 +16,7 @@ package config
 
 import (
 	"bytes"
+	"time"
 
 	"github.com/OpenIMSDK/tools/discoveryregistry"
 	"gopkg.in/yaml.v3"
@@ -266,6 +267,8 @@ type GlobalConfig struct {
 		FriendVerify *bool `yaml:"friendVerify"`
 	} `yaml:"messageVerify"`
 
+	LocalCache localCache `yaml:"localCache"`
+
 	IOSPush struct {
 		PushSound  string `yaml:"pushSound"`
 		BadgeCount bool   `yaml:"badgeCount"`
@@ -382,6 +385,33 @@ type notification struct {
 	ConversationSetPrivate NotificationConf `yaml:"conversationSetPrivate"`
 }
 
+type LocalCache struct {
+	Topic         string `yaml:"topic"`
+	SlotNum       int    `yaml:"slotNum"`
+	SlotSize      int    `yaml:"slotSize"`
+	SuccessExpire int    `yaml:"successExpire"` // second
+	FailedExpire  int    `yaml:"failedExpire"`  // second
+}
+
+func (l LocalCache) Failed() time.Duration {
+	return time.Second * time.Duration(l.FailedExpire)
+}
+
+func (l LocalCache) Success() time.Duration {
+	return time.Second * time.Duration(l.SuccessExpire)
+}
+
+func (l LocalCache) Enable() bool {
+	return l.Topic != "" && l.SlotNum > 0 && l.SlotSize > 0
+}
+
+type localCache struct {
+	User         LocalCache `yaml:"user"`
+	Group        LocalCache `yaml:"group"`
+	Friend       LocalCache `yaml:"friend"`
+	Conversation LocalCache `yaml:"conversation"`
+}
+
 func (c *GlobalConfig) GetServiceNames() []string {
 	return []string{
 		c.RpcRegisterName.OpenImUserName,

pkg/common/config/parse_test.go

@@ -16,9 +16,12 @@ package config
 
 import (
 	_ "embed"
+	"fmt"
 	"reflect"
 	"testing"
 
+	"gopkg.in/yaml.v3"
+
 	"github.com/openimsdk/open-im-server/v3/pkg/msgprocessor"
 )
 
@@ -116,3 +119,13 @@ func TestInitConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestName(t *testing.T) {
+	Config.LocalCache.Friend.Topic = "friend"
+	Config.LocalCache.Friend.SlotNum = 500
+	Config.LocalCache.Friend.SlotSize = 20000
+
+	data, _ := yaml.Marshal(&Config)
+
+	fmt.Println(string(data))
+}

pkg/common/db/cache/black.go

@@ -18,7 +18,10 @@ import (
 	"context"
 	"time"
 
+	"github.com/OpenIMSDK/tools/log"
 	"github.com/dtm-labs/rockscache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
 	"github.com/redis/go-redis/v9"
 )
@@ -47,11 +50,15 @@ type BlackCacheRedis struct {
 
 func NewBlackCacheRedis(rdb redis.UniversalClient, blackDB relationtb.BlackModelInterface, options rockscache.Options) BlackCache {
 	rcClient := rockscache.NewClient(rdb, options)
-
+	mc := NewMetaCacheRedis(rcClient)
+	b := config.Config.LocalCache.Friend
+	log.ZDebug(context.Background(), "black local cache init", "Topic", b.Topic, "SlotNum", b.SlotNum, "SlotSize", b.SlotSize, "enable", b.Enable())
+	mc.SetTopic(b.Topic)
+	mc.SetRawRedisClient(rdb)
 	return &BlackCacheRedis{
 		expireTime: blackExpireTime,
 		rcClient:   rcClient,
-		metaCache:  NewMetaCacheRedis(rcClient),
+		metaCache:  mc,
 		blackDB:    blackDB,
 	}
 }
@@ -61,12 +68,12 @@ func (b *BlackCacheRedis) NewCache() BlackCache {
 		expireTime: b.expireTime,
 		rcClient:   b.rcClient,
 		blackDB:    b.blackDB,
-		metaCache:  NewMetaCacheRedis(b.rcClient, b.metaCache.GetPreDelKeys()...),
+		metaCache:  b.Copy(),
 	}
 }
 
 func (b *BlackCacheRedis) getBlackIDsKey(ownerUserID string) string {
-	return blackIDsKey + ownerUserID
+	return cachekey.GetBlackIDsKey(ownerUserID)
 }
 
 func (b *BlackCacheRedis) GetBlackIDs(ctx context.Context, userID string) (blackIDs []string, err error) {

pkg/common/db/cache/config.go

@@ -0,0 +1,81 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cache
+
+import (
+	"strings"
+	"sync"
+
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+)
+
+var (
+	once      sync.Once
+	subscribe map[string][]string
+)
+
+func getPublishKey(topic string, key []string) []string {
+	if topic == "" || len(key) == 0 {
+		return nil
+	}
+	once.Do(func() {
+		list := []struct {
+			Local config.LocalCache
+			Keys  []string
+		}{
+			{
+				Local: config.Config.LocalCache.User,
+				Keys:  []string{cachekey.UserInfoKey, cachekey.UserGlobalRecvMsgOptKey},
+			},
+			{
+				Local: config.Config.LocalCache.Group,
+				Keys:  []string{cachekey.GroupMemberIDsKey, cachekey.GroupInfoKey, cachekey.GroupMemberInfoKey},
+			},
+			{
+				Local: config.Config.LocalCache.Friend,
+				Keys:  []string{cachekey.FriendIDsKey, cachekey.BlackIDsKey},
+			},
+			{
+				Local: config.Config.LocalCache.Conversation,
+				Keys:  []string{cachekey.ConversationKey, cachekey.ConversationIDsKey, cachekey.ConversationNotReceiveMessageUserIDsKey},
+			},
+		}
+		subscribe = make(map[string][]string)
+		for _, v := range list {
+			if v.Local.Enable() {
+				subscribe[v.Local.Topic] = v.Keys
+			}
+		}
+	})
+	prefix, ok := subscribe[topic]
+	if !ok {
+		return nil
+	}
+	res := make([]string, 0, len(key))
+	for _, k := range key {
+		var exist bool
+		for _, p := range prefix {
+			if strings.HasPrefix(k, p) {
+				exist = true
+				break
+			}
+		}
+		if exist {
+			res = append(res, k)
+		}
+	}
+	return res
+}

pkg/common/db/cache/conversation.go

@@ -20,21 +20,24 @@ import (
 	"strings"
 	"time"
 
+	"github.com/OpenIMSDK/tools/log"
 	"github.com/OpenIMSDK/tools/utils"
 	"github.com/dtm-labs/rockscache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
 	"github.com/redis/go-redis/v9"
 )
 
 const (
-	conversationKey                          = "CONVERSATION:"
-	conversationIDsKey                       = "CONVERSATION_IDS:"
-	conversationIDsHashKey                   = "CONVERSATION_IDS_HASH:"
-	conversationHasReadSeqKey                = "CONVERSATION_HAS_READ_SEQ:"
-	recvMsgOptKey                            = "RECV_MSG_OPT:"
-	superGroupRecvMsgNotNotifyUserIDsKey     = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
-	superGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
-	conversationNotReceiveMessageUserIDsKey  = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:"
+	//conversationKey                          = "CONVERSATION:"
+	//conversationIDsKey                       = "CONVERSATION_IDS:"
+	//conversationIDsHashKey                   = "CONVERSATION_IDS_HASH:"
+	//conversationHasReadSeqKey                = "CONVERSATION_HAS_READ_SEQ:"
+	//recvMsgOptKey                            = "RECV_MSG_OPT:"
+	//superGroupRecvMsgNotNotifyUserIDsKey     = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS:"
+	//superGroupRecvMsgNotNotifyUserIDsHashKey = "SUPER_GROUP_RECV_MSG_NOT_NOTIFY_USER_IDS_HASH:"
+	//conversationNotReceiveMessageUserIDsKey  = "CONVERSATION_NOT_RECEIVE_MESSAGE_USER_IDS:".
 
 	conversationExpireTime = time.Second * 60 * 60 * 12
 )
@@ -81,10 +84,14 @@ type ConversationCache interface {
 
 func NewConversationRedis(rdb redis.UniversalClient, opts rockscache.Options, db relationtb.ConversationModelInterface) ConversationCache {
 	rcClient := rockscache.NewClient(rdb, opts)
-
+	mc := NewMetaCacheRedis(rcClient)
+	c := config.Config.LocalCache.Conversation
+	log.ZDebug(context.Background(), "black local cache init", "Topic", c.Topic, "SlotNum", c.SlotNum, "SlotSize", c.SlotSize, "enable", c.Enable())
+	mc.SetTopic(c.Topic)
+	mc.SetRawRedisClient(rdb)
 	return &ConversationRedisCache{
 		rcClient:       rcClient,
-		metaCache:      NewMetaCacheRedis(rcClient),
+		metaCache:      mc,
 		conversationDB: db,
 		expireTime:     conversationExpireTime,
 	}
@@ -115,38 +122,42 @@ type ConversationRedisCache struct {
 func (c *ConversationRedisCache) NewCache() ConversationCache {
 	return &ConversationRedisCache{
 		rcClient:       c.rcClient,
-		metaCache:      NewMetaCacheRedis(c.rcClient, c.metaCache.GetPreDelKeys()...),
+		metaCache:      c.Copy(),
 		conversationDB: c.conversationDB,
 		expireTime:     c.expireTime,
 	}
 }
 
 func (c *ConversationRedisCache) getConversationKey(ownerUserID, conversationID string) string {
-	return conversationKey + ownerUserID + ":" + conversationID
+	return cachekey.GetConversationKey(ownerUserID, conversationID)
 }
 
 func (c *ConversationRedisCache) getConversationIDsKey(ownerUserID string) string {
-	return conversationIDsKey + ownerUserID
+	return cachekey.GetConversationIDsKey(ownerUserID)
 }
 
 func (c *ConversationRedisCache) getSuperGroupRecvNotNotifyUserIDsKey(groupID string) string {
-	return superGroupRecvMsgNotNotifyUserIDsKey + groupID
+	return cachekey.GetSuperGroupRecvNotNotifyUserIDsKey(groupID)
 }
 
 func (c *ConversationRedisCache) getRecvMsgOptKey(ownerUserID, conversationID string) string {
-	return recvMsgOptKey + ownerUserID + ":" + conversationID
+	return cachekey.GetRecvMsgOptKey(ownerUserID, conversationID)
 }
 
 func (c *ConversationRedisCache) getSuperGroupRecvNotNotifyUserIDsHashKey(groupID string) string {
-	return superGroupRecvMsgNotNotifyUserIDsHashKey + groupID
+	return cachekey.GetSuperGroupRecvNotNotifyUserIDsHashKey(groupID)
 }
 
 func (c *ConversationRedisCache) getConversationHasReadSeqKey(ownerUserID, conversationID string) string {
-	return conversationHasReadSeqKey + ownerUserID + ":" + conversationID
+	return cachekey.GetConversationHasReadSeqKey(ownerUserID, conversationID)
 }
 
 func (c *ConversationRedisCache) getConversationNotReceiveMessageUserIDsKey(conversationID string) string {
-	return conversationNotReceiveMessageUserIDsKey + conversationID
+	return cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID)
+}
+
+func (c *ConversationRedisCache) getUserConversationIDsHashKey(ownerUserID string) string {
+	return cachekey.GetUserConversationIDsHashKey(ownerUserID)
 }
 
 func (c *ConversationRedisCache) GetUserConversationIDs(ctx context.Context, ownerUserID string) ([]string, error) {
@@ -166,10 +177,6 @@ func (c *ConversationRedisCache) DelConversationIDs(userIDs ...string) Conversat
 	return cache
 }
 
-func (c *ConversationRedisCache) getUserConversationIDsHashKey(ownerUserID string) string {
-	return conversationIDsHashKey + ownerUserID
-}
-
 func (c *ConversationRedisCache) GetUserConversationIDsHash(ctx context.Context, ownerUserID string) (hash uint64, err error) {
 	return getCache(
 		ctx,

pkg/common/db/cache/friend.go

@@ -18,17 +18,20 @@ import (
 	"context"
 	"time"
 
+	"github.com/OpenIMSDK/tools/log"
 	"github.com/OpenIMSDK/tools/utils"
 	"github.com/dtm-labs/rockscache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
 	"github.com/redis/go-redis/v9"
 )
 
 const (
 	friendExpireTime = time.Second * 60 * 60 * 12
-	friendIDsKey        = "FRIEND_IDS:"
-	TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:"
-	friendKey           = "FRIEND_INFO:"
+	//friendIDsKey        = "FRIEND_IDS:"
+	//TwoWayFriendsIDsKey = "COMMON_FRIENDS_IDS:"
+	//friendKey           = "FRIEND_INFO:".
 )
 
 // FriendCache is an interface for caching friend-related data.
@@ -58,8 +61,13 @@ type FriendCacheRedis struct {
 func NewFriendCacheRedis(rdb redis.UniversalClient, friendDB relationtb.FriendModelInterface,
 	options rockscache.Options) FriendCache {
 	rcClient := rockscache.NewClient(rdb, options)
+	mc := NewMetaCacheRedis(rcClient)
+	f := config.Config.LocalCache.Friend
+	log.ZDebug(context.Background(), "friend local cache init", "Topic", f.Topic, "SlotNum", f.SlotNum, "SlotSize", f.SlotSize, "enable", f.Enable())
+	mc.SetTopic(f.Topic)
+	mc.SetRawRedisClient(rdb)
 	return &FriendCacheRedis{
-		metaCache:  NewMetaCacheRedis(rcClient),
+		metaCache:  mc,
 		friendDB:   friendDB,
 		expireTime: friendExpireTime,
 		rcClient:   rcClient,
@@ -70,7 +78,7 @@ func NewFriendCacheRedis(rdb redis.UniversalClient, friendDB relationtb.FriendMo
 func (f *FriendCacheRedis) NewCache() FriendCache {
 	return &FriendCacheRedis{
 		rcClient:   f.rcClient,
-		metaCache:  NewMetaCacheRedis(f.rcClient, f.metaCache.GetPreDelKeys()...),
+		metaCache:  f.Copy(),
 		friendDB:   f.friendDB,
 		expireTime: f.expireTime,
 	}
@@ -78,17 +86,17 @@ func (f *FriendCacheRedis) NewCache() FriendCache {
 
 // getFriendIDsKey returns the key for storing friend IDs in the cache.
 func (f *FriendCacheRedis) getFriendIDsKey(ownerUserID string) string {
-	return friendIDsKey + ownerUserID
+	return cachekey.GetFriendIDsKey(ownerUserID)
 }
 
 // getTwoWayFriendsIDsKey returns the key for storing two-way friend IDs in the cache.
 func (f *FriendCacheRedis) getTwoWayFriendsIDsKey(ownerUserID string) string {
-	return TwoWayFriendsIDsKey + ownerUserID
+	return cachekey.GetTwoWayFriendsIDsKey(ownerUserID)
 }
 
 // getFriendKey returns the key for storing friend info in the cache.
 func (f *FriendCacheRedis) getFriendKey(ownerUserID, friendUserID string) string {
-	return friendKey + ownerUserID + "-" + friendUserID
+	return cachekey.GetFriendKey(ownerUserID, friendUserID)
 }
 
 // GetFriendIDs retrieves friend IDs from the cache or the database if not found.

pkg/common/db/cache/group.go

@@ -17,7 +17,6 @@ package cache
 import (
 	"context"
 	"fmt"
-	"strconv"
 	"time"
 
 	"github.com/OpenIMSDK/protocol/constant"
@@ -25,20 +24,21 @@ import (
 	"github.com/OpenIMSDK/tools/log"
 	"github.com/OpenIMSDK/tools/utils"
 	"github.com/dtm-labs/rockscache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
 	"github.com/redis/go-redis/v9"
 )
 
 const (
 	groupExpireTime = time.Second * 60 * 60 * 12
-	groupInfoKey        = "GROUP_INFO:"
-	groupMemberIDsKey   = "GROUP_MEMBER_IDS:"
-	groupMembersHashKey = "GROUP_MEMBERS_HASH2:"
-	groupMemberInfoKey  = "GROUP_MEMBER_INFO:"
-	//groupOwnerInfoKey   = "GROUP_OWNER_INFO:".
-	joinedGroupsKey            = "JOIN_GROUPS_KEY:"
-	groupMemberNumKey          = "GROUP_MEMBER_NUM_CACHE:"
-	groupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:"
+	//groupInfoKey        = "GROUP_INFO:"
+	//groupMemberIDsKey   = "GROUP_MEMBER_IDS:"
+	//groupMembersHashKey = "GROUP_MEMBERS_HASH2:"
+	//groupMemberInfoKey  = "GROUP_MEMBER_INFO:"
+	//joinedGroupsKey            = "JOIN_GROUPS_KEY:"
+	//groupMemberNumKey          = "GROUP_MEMBER_NUM_CACHE:"
+	//groupRoleLevelMemberIDsKey = "GROUP_ROLE_LEVEL_MEMBER_IDS:".
 )
 
 type GroupHash interface {
@@ -101,12 +101,16 @@ func NewGroupCacheRedis(
 	opts rockscache.Options,
 ) GroupCache {
 	rcClient := rockscache.NewClient(rdb, opts)
-
+	mc := NewMetaCacheRedis(rcClient)
+	g := config.Config.LocalCache.Group
+	mc.SetTopic(g.Topic)
+	log.ZDebug(context.Background(), "group local cache init", "Topic", g.Topic, "SlotNum", g.SlotNum, "SlotSize", g.SlotSize, "enable", g.Enable())
+	mc.SetRawRedisClient(rdb)
 	return &GroupCacheRedis{
 		rcClient: rcClient, expireTime: groupExpireTime,
 		groupDB: groupDB, groupMemberDB: groupMemberDB, groupRequestDB: groupRequestDB,
 		groupHash: hashCode,
-		metaCache: NewMetaCacheRedis(rcClient),
+		metaCache: mc,
 	}
 }
 
@@ -117,36 +121,36 @@ func (g *GroupCacheRedis) NewCache() GroupCache {
 		groupDB:        g.groupDB,
 		groupMemberDB:  g.groupMemberDB,
 		groupRequestDB: g.groupRequestDB,
-		metaCache:      NewMetaCacheRedis(g.rcClient, g.metaCache.GetPreDelKeys()...),
+		metaCache:      g.Copy(),
 	}
 }
 
 func (g *GroupCacheRedis) getGroupInfoKey(groupID string) string {
-	return groupInfoKey + groupID
+	return cachekey.GetGroupInfoKey(groupID)
 }
 
 func (g *GroupCacheRedis) getJoinedGroupsKey(userID string) string {
-	return joinedGroupsKey + userID
+	return cachekey.GetJoinedGroupsKey(userID)
 }
 
 func (g *GroupCacheRedis) getGroupMembersHashKey(groupID string) string {
-	return groupMembersHashKey + groupID
+	return cachekey.GetGroupMembersHashKey(groupID)
 }
 
 func (g *GroupCacheRedis) getGroupMemberIDsKey(groupID string) string {
-	return groupMemberIDsKey + groupID
+	return cachekey.GetGroupMemberIDsKey(groupID)
 }
 
 func (g *GroupCacheRedis) getGroupMemberInfoKey(groupID, userID string) string {
-	return groupMemberInfoKey + groupID + "-" + userID
+	return cachekey.GetGroupMemberInfoKey(groupID, userID)
 }
 
 func (g *GroupCacheRedis) getGroupMemberNumKey(groupID string) string {
-	return groupMemberNumKey + groupID
+	return cachekey.GetGroupMemberNumKey(groupID)
 }
 
 func (g *GroupCacheRedis) getGroupRoleLevelMemberIDsKey(groupID string, roleLevel int32) string {
-	return groupRoleLevelMemberIDsKey + groupID + "-" + strconv.Itoa(int(roleLevel))
+	return cachekey.GetGroupRoleLevelMemberIDsKey(groupID, roleLevel)
 }
 
 func (g *GroupCacheRedis) GetGroupIndex(group *relationtb.GroupModel, keys []string) (int, error) {

pkg/common/db/cache/meta_cache.go

@@ -26,6 +26,7 @@ import (
 	"github.com/OpenIMSDK/tools/mw/specialerror"
 	"github.com/OpenIMSDK/tools/utils"
 	"github.com/dtm-labs/rockscache"
+	"github.com/redis/go-redis/v9"
 )
 
 const (
@@ -43,6 +44,9 @@ type metaCache interface {
 	AddKeys(keys ...string)
 	ClearKeys()
 	GetPreDelKeys() []string
+	SetTopic(topic string)
+	SetRawRedisClient(cli redis.UniversalClient)
+	Copy() metaCache
 }
 
 func NewMetaCacheRedis(rcClient *rockscache.Client, keys ...string) metaCache {
@@ -50,10 +54,36 @@ func NewMetaCacheRedis(rcClient *rockscache.Client, keys ...string) metaCache {
 }
 
 type metaCacheRedis struct {
+	topic         string
 	rcClient      *rockscache.Client
 	keys          []string
 	maxRetryTimes int
 	retryInterval time.Duration
+	redisClient   redis.UniversalClient
+}
+
+func (m *metaCacheRedis) Copy() metaCache {
+	var keys []string
+	if len(m.keys) > 0 {
+		keys = make([]string, 0, len(m.keys)*2)
+		keys = append(keys, m.keys...)
+	}
+	return &metaCacheRedis{
+		topic:         m.topic,
+		rcClient:      m.rcClient,
+		keys:          keys,
+		maxRetryTimes: m.maxRetryTimes,
+		retryInterval: m.retryInterval,
+		redisClient:   redisClient,
+	}
+}
+
+func (m *metaCacheRedis) SetTopic(topic string) {
+	m.topic = topic
+}
+
+func (m *metaCacheRedis) SetRawRedisClient(cli redis.UniversalClient) {
+	m.redisClient = cli
 }
 
 func (m *metaCacheRedis) ExecDel(ctx context.Context, distinct ...bool) error {
@@ -61,7 +91,7 @@ func (m *metaCacheRedis) ExecDel(ctx context.Context, distinct ...bool) error {
 		m.keys = utils.Distinct(m.keys)
 	}
 	if len(m.keys) > 0 {
-		log.ZDebug(ctx, "delete cache", "keys", m.keys)
+		log.ZDebug(ctx, "delete cache", "topic", m.topic, "keys", m.keys)
 		for _, key := range m.keys {
 			for i := 0; i < m.maxRetryTimes; i++ {
 				if err := m.rcClient.TagAsDeleted(key); err != nil {
@@ -71,31 +101,18 @@ func (m *metaCacheRedis) ExecDel(ctx context.Context, distinct ...bool) error {
 				}
 				break
 			}
-
-			//retryTimes := 0
-			//for {
-			//	m.rcClient.TagAsDeleted()
-			//	if err := m.rcClient.TagAsDeletedBatch2(ctx, []string{key}); err != nil {
-			//		if retryTimes >= m.maxRetryTimes {
-			//			err = errs.ErrInternalServer.Wrap(
-			//				fmt.Sprintf(
-			//					"delete cache error: %v, keys: %v, retry times %d, please check redis server",
-			//					err,
-			//					key,
-			//					retryTimes,
-			//				),
-			//			)
-			//			log.ZWarn(ctx, "delete cache failed, please handle keys", err, "keys", key)
-			//			return err
-			//		}
-			//		retryTimes++
-			//	} else {
-			//		break
-			//	}
-			//}
+		}
+		if pk := getPublishKey(m.topic, m.keys); len(pk) > 0 {
+			data, err := json.Marshal(pk)
+			if err != nil {
+				log.ZError(ctx, "keys json marshal failed", err, "topic", m.topic, "keys", pk)
+			} else {
+				if err := m.redisClient.Publish(ctx, m.topic, string(data)).Err(); err != nil {
+					log.ZError(ctx, "redis publish cache delete error", err, "topic", m.topic, "keys", pk)
+				}
+			}
 		}
 	}
-
 	return nil
 }
 

pkg/common/db/cache/msg.go

@@ -38,11 +38,11 @@ const (
 	conversationUserMinSeq = "CON_USER_MIN_SEQ:"
 	hasReadSeq             = "HAS_READ_SEQ:"
 
-	appleDeviceToken = "DEVICE_TOKEN"
+	//appleDeviceToken = "DEVICE_TOKEN".
 	getuiToken  = "GETUI_TOKEN"
 	getuiTaskID = "GETUI_TASK_ID"
-	signalCache      = "SIGNAL_CACHE:"
-	signalListCache  = "SIGNAL_LIST_CACHE:"
+	//signalCache      = "SIGNAL_CACHE:"
+	//signalListCache  = "SIGNAL_LIST_CACHE:".
 	FCM_TOKEN = "FCM_TOKEN:"
 
 	messageCache            = "MESSAGE_CACHE:"
@@ -143,6 +143,10 @@ func (c *msgCache) getHasReadSeqKey(conversationID string, userID string) string
 	return hasReadSeq + userID + ":" + conversationID
 }
 
+func (c *msgCache) getConversationUserMinSeqKey(conversationID, userID string) string {
+	return conversationUserMinSeq + conversationID + "u:" + userID
+}
+
 func (c *msgCache) setSeq(ctx context.Context, conversationID string, seq int64, getkey func(conversationID string) string) error {
 	return errs.Wrap(c.rdb.Set(ctx, getkey(conversationID), seq, 0).Err())
 }
@@ -208,10 +212,6 @@ func (c *msgCache) GetMinSeq(ctx context.Context, conversationID string) (int64,
 	return c.getSeq(ctx, conversationID, c.getMinSeqKey)
 }
 
-func (c *msgCache) getConversationUserMinSeqKey(conversationID, userID string) string {
-	return conversationUserMinSeq + conversationID + "u:" + userID
-}
-
 func (c *msgCache) GetConversationUserMinSeq(ctx context.Context, conversationID string, userID string) (int64, error) {
 	val, err := c.rdb.Get(ctx, c.getConversationUserMinSeqKey(conversationID, userID)).Int64()
 	if err != nil {

pkg/common/db/cache/user.go

@@ -27,13 +27,15 @@ import (
 	"github.com/OpenIMSDK/tools/errs"
 	"github.com/OpenIMSDK/tools/log"
 	"github.com/dtm-labs/rockscache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/db/table/relation"
 	"github.com/redis/go-redis/v9"
 )
 
 const (
 	userExpireTime = time.Second * 60 * 60 * 12
-	userInfoKey               = "USER_INFO:"
+	//userInfoKey               = "USER_INFO:".
 	userGlobalRecvMsgOptKey   = "USER_GLOBAL_RECV_MSG_OPT_KEY:"
 	olineStatusKey            = "ONLINE_STATUS:"
 	userOlineStatusExpireTime = time.Second * 60 * 60 * 24
@@ -68,7 +70,11 @@ func NewUserCacheRedis(
 	options rockscache.Options,
 ) UserCache {
 	rcClient := rockscache.NewClient(rdb, options)
-
+	mc := NewMetaCacheRedis(rcClient)
+	u := config.Config.LocalCache.User
+	log.ZDebug(context.Background(), "user local cache init", "Topic", u.Topic, "SlotNum", u.SlotNum, "SlotSize", u.SlotSize, "enable", u.Enable())
+	mc.SetTopic(u.Topic)
+	mc.SetRawRedisClient(rdb)
 	return &UserCacheRedis{
 		rdb:        rdb,
 		metaCache:  NewMetaCacheRedis(rcClient),
@@ -81,7 +87,7 @@ func NewUserCacheRedis(
 func (u *UserCacheRedis) NewCache() UserCache {
 	return &UserCacheRedis{
 		rdb:        u.rdb,
-		metaCache:  NewMetaCacheRedis(u.rcClient, u.metaCache.GetPreDelKeys()...),
+		metaCache:  u.Copy(),
 		userDB:     u.userDB,
 		expireTime: u.expireTime,
 		rcClient:   u.rcClient,
@@ -89,18 +95,17 @@ func (u *UserCacheRedis) NewCache() UserCache {
 }
 
 func (u *UserCacheRedis) getUserInfoKey(userID string) string {
-	return userInfoKey + userID
+	return cachekey.GetUserInfoKey(userID)
 }
 
 func (u *UserCacheRedis) getUserGlobalRecvMsgOptKey(userID string) string {
-	return userGlobalRecvMsgOptKey + userID
+	return cachekey.GetUserGlobalRecvMsgOptKey(userID)
 }
 
 func (u *UserCacheRedis) GetUserInfo(ctx context.Context, userID string) (userInfo *relationtb.UserModel, err error) {
 	return getCache(ctx, u.rcClient, u.getUserInfoKey(userID), u.expireTime, func(ctx context.Context) (*relationtb.UserModel, error) {
 		return u.userDB.Take(ctx, userID)
-	},
-	)
+	})
 }
 
 func (u *UserCacheRedis) GetUsersInfo(ctx context.Context, userIDs []string) ([]*relationtb.UserModel, error) {

pkg/common/db/localcache/conversation.go

@@ -1,86 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package localcache
-
-import (
-	"context"
-	"sync"
-
-	"github.com/OpenIMSDK/protocol/conversation"
-	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
-)
-
-type ConversationLocalCache struct {
-	lock                              sync.Mutex
-	superGroupRecvMsgNotNotifyUserIDs map[string]Hash
-	conversationIDs                   map[string]Hash
-	client                            *rpcclient.ConversationRpcClient
-}
-
-type Hash struct {
-	hash uint64
-	ids  []string
-}
-
-func NewConversationLocalCache(client *rpcclient.ConversationRpcClient) *ConversationLocalCache {
-	return &ConversationLocalCache{
-		superGroupRecvMsgNotNotifyUserIDs: make(map[string]Hash),
-		conversationIDs:                   make(map[string]Hash),
-		client:                            client,
-	}
-}
-
-func (g *ConversationLocalCache) GetRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) {
-	resp, err := g.client.Client.GetRecvMsgNotNotifyUserIDs(ctx, &conversation.GetRecvMsgNotNotifyUserIDsReq{
-		GroupID: groupID,
-	})
-	if err != nil {
-		return nil, err
-	}
-	return resp.UserIDs, nil
-}
-
-func (g *ConversationLocalCache) GetConversationIDs(ctx context.Context, userID string) ([]string, error) {
-	resp, err := g.client.Client.GetUserConversationIDsHash(ctx, &conversation.GetUserConversationIDsHashReq{
-		OwnerUserID: userID,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	g.lock.Lock()
-	hash, ok := g.conversationIDs[userID]
-	g.lock.Unlock()
-
-	if !ok || hash.hash != resp.Hash {
-		conversationIDsResp, err := g.client.Client.GetConversationIDs(ctx, &conversation.GetConversationIDsReq{
-			UserID: userID,
-		})
-		if err != nil {
-			return nil, err
-		}
-
-		g.lock.Lock()
-		defer g.lock.Unlock()
-		g.conversationIDs[userID] = Hash{
-			hash: resp.Hash,
-			ids:  conversationIDsResp.ConversationIDs,
-		}
-
-		return conversationIDsResp.ConversationIDs, nil
-	}
-
-	return hash.ids, nil
-}

pkg/common/db/localcache/doc.go

@@ -1,15 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package localcache // import "github.com/openimsdk/open-im-server/v3/pkg/common/db/localcache"

pkg/common/db/localcache/group.go

@@ -1,77 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package localcache
-
-import (
-	"context"
-	"sync"
-
-	"github.com/OpenIMSDK/protocol/group"
-	"github.com/OpenIMSDK/tools/errs"
-	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
-)
-
-type GroupLocalCache struct {
-	lock   sync.Mutex
-	cache  map[string]GroupMemberIDsHash
-	client *rpcclient.GroupRpcClient
-}
-
-type GroupMemberIDsHash struct {
-	memberListHash uint64
-	userIDs        []string
-}
-
-func NewGroupLocalCache(client *rpcclient.GroupRpcClient) *GroupLocalCache {
-	return &GroupLocalCache{
-		cache:  make(map[string]GroupMemberIDsHash, 0),
-		client: client,
-	}
-}
-
-func (g *GroupLocalCache) GetGroupMemberIDs(ctx context.Context, groupID string) ([]string, error) {
-	resp, err := g.client.Client.GetGroupAbstractInfo(ctx, &group.GetGroupAbstractInfoReq{
-		GroupIDs: []string{groupID},
-	})
-	if err != nil {
-		return nil, err
-	}
-	if len(resp.GroupAbstractInfos) < 1 {
-		return nil, errs.ErrGroupIDNotFound
-	}
-
-	g.lock.Lock()
-	localHashInfo, ok := g.cache[groupID]
-	if ok && localHashInfo.memberListHash == resp.GroupAbstractInfos[0].GroupMemberListHash {
-		g.lock.Unlock()
-		return localHashInfo.userIDs, nil
-	}
-	g.lock.Unlock()
-
-	groupMembersResp, err := g.client.Client.GetGroupMemberUserIDs(ctx, &group.GetGroupMemberUserIDsReq{
-		GroupID: groupID,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	g.lock.Lock()
-	defer g.lock.Unlock()
-	g.cache[groupID] = GroupMemberIDsHash{
-		memberListHash: resp.GroupAbstractInfos[0].GroupMemberListHash,
-		userIDs:        groupMembersResp.UserIDs,
-	}
-	return g.cache[groupID].userIDs, nil
-}

pkg/common/db/mgo/conversation.go

@@ -96,8 +96,14 @@ func (c *ConversationMgo) FindUserIDAllConversations(ctx context.Context, userID
 	return mgoutil.Find[*relation.ConversationModel](ctx, c.coll, bson.M{"owner_user_id": userID})
 }
 
-func (c *ConversationMgo) FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error) {
-	return mgoutil.Find[string](ctx, c.coll, bson.M{"group_id": groupID, "recv_msg_opt": constant.ReceiveNotNotifyMessage}, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}))
+func (c *ConversationMgo) FindRecvMsgUserIDs(ctx context.Context, conversationID string, recvOpts []int) ([]string, error) {
+	var filter any
+	if len(recvOpts) == 0 {
+		filter = bson.M{"conversation_id": conversationID}
+	} else {
+		filter = bson.M{"conversation_id": conversationID, "recv_msg_opt": bson.M{"$in": recvOpts}}
+	}
+	return mgoutil.Find[string](ctx, c.coll, filter, options.Find().SetProjection(bson.M{"_id": 0, "owner_user_id": 1}))
 }
 
 func (c *ConversationMgo) GetUserRecvMsgOpt(ctx context.Context, ownerUserID, conversationID string) (opt int, err error) {

pkg/common/db/mgo/group.go

@@ -16,6 +16,7 @@ package mgo
 
 import (
 	"context"
+	"github.com/OpenIMSDK/protocol/constant"
 	"time"
 
 	"github.com/OpenIMSDK/tools/errs"
@@ -69,7 +70,8 @@ func (g *GroupMgo) Take(ctx context.Context, groupID string) (group *relation.Gr
 }
 
 func (g *GroupMgo) Search(ctx context.Context, keyword string, pagination pagination.Pagination) (total int64, groups []*relation.GroupModel, err error) {
-	return mgoutil.FindPage[*relation.GroupModel](ctx, g.coll, bson.M{"group_name": bson.M{"$regex": keyword}}, pagination)
+	return mgoutil.FindPage[*relation.GroupModel](ctx, g.coll, bson.M{"group_name": bson.M{"$regex": keyword},
+		"status": bson.M{"$ne": constant.GroupStatusDismissed}}, pagination)
 }
 
 func (g *GroupMgo) CountTotal(ctx context.Context, before *time.Time) (count int64, err error) {

pkg/common/db/table/relation/conversation.go

@@ -53,7 +53,7 @@ type ConversationModelInterface interface {
 	Take(ctx context.Context, userID, conversationID string) (conversation *ConversationModel, err error)
 	FindConversationID(ctx context.Context, userID string, conversationIDs []string) (existConversationID []string, err error)
 	FindUserIDAllConversations(ctx context.Context, userID string) (conversations []*ConversationModel, err error)
-	FindRecvMsgNotNotifyUserIDs(ctx context.Context, groupID string) ([]string, error)
+	FindRecvMsgUserIDs(ctx context.Context, conversationID string, recvOpts []int) ([]string, error)
 	GetUserRecvMsgOpt(ctx context.Context, ownerUserID, conversationID string) (opt int, err error)
 	GetAllConversationIDs(ctx context.Context) ([]string, error)
 	GetAllConversationIDsNumber(ctx context.Context) (int64, error)

pkg/common/db/unrelation/msg.go

@@ -1063,8 +1063,8 @@ func (m *MsgMongoDriver) searchMessage(ctx context.Context, req *msg.SearchMessa
 		// Changed to keyed fields for bson.M to avoid govet errors
 		condition = append(condition, bson.M{"$eq": bson.A{bson.M{"$dateToString": bson.M{"format": "%Y-%m-%d", "date": bson.M{"$toDate": "$$item.msg.send_time"}}}, req.SendTime}})
 	}
-	if req.MsgType != 0 {
-		condition = append(condition, bson.M{"$eq": bson.A{"$$item.msg.content_type", req.MsgType}})
+	if req.ContentType != 0 {
+		condition = append(condition, bson.M{"$eq": bson.A{"$$item.msg.content_type", req.ContentType}})
 	}
 	if req.SessionType != 0 {
 		condition = append(condition, bson.M{"$eq": bson.A{"$$item.msg.session_type", req.SessionType}})

pkg/common/redispubsub/redispubliser.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 OpenIM open source community. All rights reserved.
+// Copyright © 2024 OpenIM. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package admin
+package redispubsub
 
-import (
-	"time"
-)
+import "github.com/redis/go-redis/v9"
 
-// RegisterAddFriend Indicates the default friend when registering.
-type RegisterAddFriend struct {
-	UserID     string    `gorm:"column:user_id;primary_key;type:char(64)"`
-	CreateTime time.Time `gorm:"column:create_time"`
+type Publisher struct {
+	client  redis.UniversalClient
+	channel string
 }
 
-func (RegisterAddFriend) TableName() string {
-	return "register_add_friends"
+func NewPublisher(client redis.UniversalClient, channel string) *Publisher {
+	return &Publisher{client: client, channel: channel}
+}
+
+func (p *Publisher) Publish(message string) error {
+	return p.client.Publish(ctx, p.channel, message).Err()
 }

pkg/common/redispubsub/redissubscriber.go

@@ -0,0 +1,49 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package redispubsub
+
+import (
+	"context"
+
+	"github.com/redis/go-redis/v9"
+)
+
+var ctx = context.Background()
+
+type Subscriber struct {
+	client  redis.UniversalClient
+	channel string
+}
+
+func NewSubscriber(client redis.UniversalClient, channel string) *Subscriber {
+	return &Subscriber{client: client, channel: channel}
+}
+
+func (s *Subscriber) OnMessage(ctx context.Context, callback func(string)) error {
+	messageChannel := s.client.Subscribe(ctx, s.channel).Channel()
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case msg := <-messageChannel:
+				callback(msg.Payload)
+			}
+		}
+	}()
+
+	return nil
+}

pkg/localcache/cache.go

@@ -0,0 +1,127 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package localcache
+
+import (
+	"context"
+	"hash/fnv"
+	"unsafe"
+
+	"github.com/openimsdk/localcache/link"
+	"github.com/openimsdk/localcache/lru"
+)
+
+type Cache[V any] interface {
+	Get(ctx context.Context, key string, fetch func(ctx context.Context) (V, error)) (V, error)
+	GetLink(ctx context.Context, key string, fetch func(ctx context.Context) (V, error), link ...string) (V, error)
+	Del(ctx context.Context, key ...string)
+	DelLocal(ctx context.Context, key ...string)
+	Stop()
+}
+
+func New[V any](opts ...Option) Cache[V] {
+	opt := defaultOption()
+	for _, o := range opts {
+		o(opt)
+	}
+
+	c := cache[V]{opt: opt}
+	if opt.localSlotNum > 0 && opt.localSlotSize > 0 {
+		createSimpleLRU := func() lru.LRU[string, V] {
+			if opt.expirationEvict {
+				return lru.NewExpirationLRU[string, V](opt.localSlotSize, opt.localSuccessTTL, opt.localFailedTTL, opt.target, c.onEvict)
+			} else {
+				return lru.NewLayLRU[string, V](opt.localSlotSize, opt.localSuccessTTL, opt.localFailedTTL, opt.target, c.onEvict)
+			}
+		}
+		if opt.localSlotNum == 1 {
+			c.local = createSimpleLRU()
+		} else {
+			c.local = lru.NewSlotLRU[string, V](opt.localSlotNum, func(key string) uint64 {
+				h := fnv.New64a()
+				h.Write(*(*[]byte)(unsafe.Pointer(&key)))
+				return h.Sum64()
+			}, createSimpleLRU)
+		}
+		if opt.linkSlotNum > 0 {
+			c.link = link.New(opt.linkSlotNum)
+		}
+	}
+	return &c
+}
+
+type cache[V any] struct {
+	opt   *option
+	link  link.Link
+	local lru.LRU[string, V]
+}
+
+func (c *cache[V]) onEvict(key string, value V) {
+	if c.link != nil {
+		lks := c.link.Del(key)
+		for k := range lks {
+			if key != k { // prevent deadlock
+				c.local.Del(k)
+			}
+		}
+	}
+}
+
+func (c *cache[V]) del(key ...string) {
+	if c.local == nil {
+		return
+	}
+	for _, k := range key {
+		c.local.Del(k)
+		if c.link != nil {
+			lks := c.link.Del(k)
+			for k := range lks {
+				c.local.Del(k)
+			}
+		}
+	}
+}
+
+func (c *cache[V]) Get(ctx context.Context, key string, fetch func(ctx context.Context) (V, error)) (V, error) {
+	return c.GetLink(ctx, key, fetch)
+}
+
+func (c *cache[V]) GetLink(ctx context.Context, key string, fetch func(ctx context.Context) (V, error), link ...string) (V, error) {
+	if c.local != nil {
+		return c.local.Get(key, func() (V, error) {
+			if len(link) > 0 {
+				c.link.Link(key, link...)
+			}
+			return fetch(ctx)
+		})
+	} else {
+		return fetch(ctx)
+	}
+}
+
+func (c *cache[V]) Del(ctx context.Context, key ...string) {
+	for _, fn := range c.opt.delFn {
+		fn(ctx, key...)
+	}
+	c.del(key...)
+}
+
+func (c *cache[V]) DelLocal(ctx context.Context, key ...string) {
+	c.del(key...)
+}
+
+func (c *cache[V]) Stop() {
+	c.local.Stop()
+}

pkg/localcache/cache_test.go

@@ -0,0 +1,93 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package localcache
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestName(t *testing.T) {
+	c := New[string](WithExpirationEvict())
+	//c := New[string]()
+	ctx := context.Background()
+
+	const (
+		num  = 10000
+		tNum = 10000
+		kNum = 100000
+		pNum = 100
+	)
+
+	getKey := func(v uint64) string {
+		return fmt.Sprintf("key_%d", v%kNum)
+	}
+
+	start := time.Now()
+	t.Log("start", start)
+
+	var (
+		get atomic.Int64
+		del atomic.Int64
+	)
+
+	incrGet := func() {
+		if v := get.Add(1); v%pNum == 0 {
+			//t.Log("#get count", v/pNum)
+		}
+	}
+	incrDel := func() {
+		if v := del.Add(1); v%pNum == 0 {
+			//t.Log("@del count", v/pNum)
+		}
+	}
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < tNum; i++ {
+		wg.Add(2)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < num; i++ {
+				c.Get(ctx, getKey(rand.Uint64()), func(ctx context.Context) (string, error) {
+					return fmt.Sprintf("index_%d", i), nil
+				})
+				incrGet()
+			}
+		}()
+
+		go func() {
+			defer wg.Done()
+			time.Sleep(time.Second / 10)
+			for i := 0; i < num; i++ {
+				c.Del(ctx, getKey(rand.Uint64()))
+				incrDel()
+			}
+		}()
+	}
+
+	wg.Wait()
+	end := time.Now()
+	t.Log("end", end)
+	t.Log("time", end.Sub(start))
+	t.Log("get", get.Load())
+	t.Log("del", del.Load())
+	// 137.35s
+}

pkg/localcache/go.mod

@@ -0,0 +1,5 @@
+module github.com/openimsdk/localcache
+
+go 1.19
+
+require github.com/hashicorp/golang-lru/v2 v2.0.7

pkg/localcache/link/link.go

@@ -0,0 +1,123 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package link
+
+import (
+	"hash/fnv"
+	"sync"
+	"unsafe"
+)
+
+type Link interface {
+	Link(key string, link ...string)
+	Del(key string) map[string]struct{}
+}
+
+func newLinkKey() *linkKey {
+	return &linkKey{
+		data: make(map[string]map[string]struct{}),
+	}
+}
+
+type linkKey struct {
+	lock sync.Mutex
+	data map[string]map[string]struct{}
+}
+
+func (x *linkKey) link(key string, link ...string) {
+	x.lock.Lock()
+	defer x.lock.Unlock()
+	v, ok := x.data[key]
+	if !ok {
+		v = make(map[string]struct{})
+		x.data[key] = v
+	}
+	for _, k := range link {
+		v[k] = struct{}{}
+	}
+}
+
+func (x *linkKey) del(key string) map[string]struct{} {
+	x.lock.Lock()
+	defer x.lock.Unlock()
+	ks, ok := x.data[key]
+	if !ok {
+		return nil
+	}
+	delete(x.data, key)
+	return ks
+}
+
+func New(n int) Link {
+	if n <= 0 {
+		panic("must be greater than 0")
+	}
+	slots := make([]*linkKey, n)
+	for i := 0; i < len(slots); i++ {
+		slots[i] = newLinkKey()
+	}
+	return &slot{
+		n:     uint64(n),
+		slots: slots,
+	}
+}
+
+type slot struct {
+	n     uint64
+	slots []*linkKey
+}
+
+func (x *slot) index(s string) uint64 {
+	h := fnv.New64a()
+	_, _ = h.Write(*(*[]byte)(unsafe.Pointer(&s)))
+	return h.Sum64() % x.n
+}
+
+func (x *slot) Link(key string, link ...string) {
+	if len(link) == 0 {
+		return
+	}
+	mk := key
+	lks := make([]string, len(link))
+	for i, k := range link {
+		lks[i] = k
+	}
+	x.slots[x.index(mk)].link(mk, lks...)
+	for _, lk := range lks {
+		x.slots[x.index(lk)].link(lk, mk)
+	}
+}
+
+func (x *slot) Del(key string) map[string]struct{} {
+	return x.delKey(key)
+}
+
+func (x *slot) delKey(k string) map[string]struct{} {
+	del := make(map[string]struct{})
+	stack := []string{k}
+	for len(stack) > 0 {
+		curr := stack[len(stack)-1]
+		stack = stack[:len(stack)-1]
+		if _, ok := del[curr]; ok {
+			continue
+		}
+		del[curr] = struct{}{}
+		childKeys := x.slots[x.index(curr)].del(curr)
+		for ck := range childKeys {
+			stack = append(stack, ck)
+		}
+	}
+	return del
+}

pkg/localcache/link/link_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 OpenIM. All rights reserved.
+// Copyright © 2024 OpenIM. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,17 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package utils
+package link
 
-import "time"
+import (
+	"testing"
+)
+
+func TestName(t *testing.T) {
+
+	v := New(1)
+
+	//v.Link("a:1", "b:1", "c:1", "d:1")
+	v.Link("a:1", "b:1", "c:1")
+	v.Link("z:1", "b:1")
+
+	//v.DelKey("a:1")
+	v.Del("z:1")
+
+	t.Log(v)
 
-func InitTime(ts ...*time.Time) {
-	for i := range ts {
-		if ts[i] == nil {
-			continue
-		}
-		if ts[i].IsZero() || ts[i].UnixMicro() < 0 {
-			*ts[i] = time.UnixMicro(0)
-		}
-	}
 }

pkg/localcache/lru/lru.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 OpenIM. All rights reserved.
+// Copyright © 2024 OpenIM. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,25 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package relation
+package lru
 
-import (
-	"gorm.io/gorm"
+import "github.com/hashicorp/golang-lru/v2/simplelru"
 
-	"github.com/OpenIMSDK/tools/utils"
-)
+type EvictCallback[K comparable, V any] simplelru.EvictCallback[K, V]
 
-type BatchUpdateGroupMember struct {
-	GroupID string
-	UserID  string
-	Map     map[string]any
+type LRU[K comparable, V any] interface {
+	Get(key K, fetch func() (V, error)) (V, error)
+	Del(key K) bool
+	Stop()
 }
 
-type GroupSimpleUserID struct {
-	Hash      uint64
-	MemberNum uint32
-}
+type Target interface {
+	IncrGetHit()
+	IncrGetSuccess()
+	IncrGetFailed()
 
-func IsNotFound(err error) bool {
-	return utils.Unwrap(err) == gorm.ErrRecordNotFound
+	IncrDelHit()
+	IncrDelNotFound()
 }

pkg/localcache/lru/lru_expiration.go

@@ -0,0 +1,93 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lru
+
+import (
+	"sync"
+	"time"
+
+	"github.com/hashicorp/golang-lru/v2/expirable"
+)
+
+func NewExpirationLRU[K comparable, V any](size int, successTTL, failedTTL time.Duration, target Target, onEvict EvictCallback[K, V]) LRU[K, V] {
+	var cb expirable.EvictCallback[K, *expirationLruItem[V]]
+	if onEvict != nil {
+		cb = func(key K, value *expirationLruItem[V]) {
+			onEvict(key, value.value)
+		}
+	}
+	core := expirable.NewLRU[K, *expirationLruItem[V]](size, cb, successTTL)
+	return &ExpirationLRU[K, V]{
+		core:       core,
+		successTTL: successTTL,
+		failedTTL:  failedTTL,
+		target:     target,
+	}
+}
+
+type expirationLruItem[V any] struct {
+	lock  sync.RWMutex
+	err   error
+	value V
+}
+
+type ExpirationLRU[K comparable, V any] struct {
+	lock       sync.Mutex
+	core       *expirable.LRU[K, *expirationLruItem[V]]
+	successTTL time.Duration
+	failedTTL  time.Duration
+	target     Target
+}
+
+func (x *ExpirationLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) {
+	x.lock.Lock()
+	v, ok := x.core.Get(key)
+	if ok {
+		x.lock.Unlock()
+		x.target.IncrGetSuccess()
+		v.lock.RLock()
+		defer v.lock.RUnlock()
+		return v.value, v.err
+	} else {
+		v = &expirationLruItem[V]{}
+		x.core.Add(key, v)
+		v.lock.Lock()
+		x.lock.Unlock()
+		defer v.lock.Unlock()
+		v.value, v.err = fetch()
+		if v.err == nil {
+			x.target.IncrGetSuccess()
+		} else {
+			x.target.IncrGetFailed()
+			x.core.Remove(key)
+		}
+		return v.value, v.err
+	}
+}
+
+func (x *ExpirationLRU[K, V]) Del(key K) bool {
+	x.lock.Lock()
+	ok := x.core.Remove(key)
+	x.lock.Unlock()
+	if ok {
+		x.target.IncrDelHit()
+	} else {
+		x.target.IncrDelNotFound()
+	}
+	return ok
+}
+
+func (x *ExpirationLRU[K, V]) Stop() {
+}

pkg/localcache/lru/lru_lazy.go

@@ -0,0 +1,105 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lru
+
+import (
+	"sync"
+	"time"
+
+	"github.com/hashicorp/golang-lru/v2/simplelru"
+)
+
+type layLruItem[V any] struct {
+	lock    sync.Mutex
+	expires int64
+	err     error
+	value   V
+}
+
+func NewLayLRU[K comparable, V any](size int, successTTL, failedTTL time.Duration, target Target, onEvict EvictCallback[K, V]) *LayLRU[K, V] {
+	var cb simplelru.EvictCallback[K, *layLruItem[V]]
+	if onEvict != nil {
+		cb = func(key K, value *layLruItem[V]) {
+			onEvict(key, value.value)
+		}
+	}
+	core, err := simplelru.NewLRU[K, *layLruItem[V]](size, cb)
+	if err != nil {
+		panic(err)
+	}
+	return &LayLRU[K, V]{
+		core:       core,
+		successTTL: successTTL,
+		failedTTL:  failedTTL,
+		target:     target,
+	}
+}
+
+type LayLRU[K comparable, V any] struct {
+	lock       sync.Mutex
+	core       *simplelru.LRU[K, *layLruItem[V]]
+	successTTL time.Duration
+	failedTTL  time.Duration
+	target     Target
+}
+
+func (x *LayLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) {
+	x.lock.Lock()
+	v, ok := x.core.Get(key)
+	if ok {
+		x.lock.Unlock()
+		v.lock.Lock()
+		expires, value, err := v.expires, v.value, v.err
+		if expires != 0 && expires > time.Now().UnixMilli() {
+			v.lock.Unlock()
+			x.target.IncrGetHit()
+			return value, err
+		}
+	} else {
+		v = &layLruItem[V]{}
+		x.core.Add(key, v)
+		v.lock.Lock()
+		x.lock.Unlock()
+	}
+	defer v.lock.Unlock()
+	if v.expires > time.Now().UnixMilli() {
+		return v.value, v.err
+	}
+	v.value, v.err = fetch()
+	if v.err == nil {
+		v.expires = time.Now().Add(x.successTTL).UnixMilli()
+		x.target.IncrGetSuccess()
+	} else {
+		v.expires = time.Now().Add(x.failedTTL).UnixMilli()
+		x.target.IncrGetFailed()
+	}
+	return v.value, v.err
+}
+
+func (x *LayLRU[K, V]) Del(key K) bool {
+	x.lock.Lock()
+	ok := x.core.Remove(key)
+	x.lock.Unlock()
+	if ok {
+		x.target.IncrDelHit()
+	} else {
+		x.target.IncrDelNotFound()
+	}
+	return ok
+}
+
+func (x *LayLRU[K, V]) Stop() {
+
+}

pkg/localcache/lru/lru_lazy_test.go

@@ -0,0 +1,118 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lru
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+	"unsafe"
+)
+
+type cacheTarget struct {
+	getHit      int64
+	getSuccess  int64
+	getFailed   int64
+	delHit      int64
+	delNotFound int64
+}
+
+func (r *cacheTarget) IncrGetHit() {
+	atomic.AddInt64(&r.getHit, 1)
+}
+
+func (r *cacheTarget) IncrGetSuccess() {
+	atomic.AddInt64(&r.getSuccess, 1)
+}
+
+func (r *cacheTarget) IncrGetFailed() {
+	atomic.AddInt64(&r.getFailed, 1)
+}
+
+func (r *cacheTarget) IncrDelHit() {
+	atomic.AddInt64(&r.delHit, 1)
+}
+
+func (r *cacheTarget) IncrDelNotFound() {
+	atomic.AddInt64(&r.delNotFound, 1)
+}
+
+func (r *cacheTarget) String() string {
+	return fmt.Sprintf("getHit: %d, getSuccess: %d, getFailed: %d, delHit: %d, delNotFound: %d", r.getHit, r.getSuccess, r.getFailed, r.delHit, r.delNotFound)
+}
+
+func TestName(t *testing.T) {
+	target := &cacheTarget{}
+	l := NewSlotLRU[string, string](100, func(k string) uint64 {
+		h := fnv.New64a()
+		h.Write(*(*[]byte)(unsafe.Pointer(&k)))
+		return h.Sum64()
+	}, func() LRU[string, string] {
+		return NewExpirationLRU[string, string](100, time.Second*60, time.Second, target, nil)
+	})
+	//l := NewInertiaLRU[string, string](1000, time.Second*20, time.Second*5, target)
+
+	fn := func(key string, n int, fetch func() (string, error)) {
+		for i := 0; i < n; i++ {
+			//v, err := l.Get(key, fetch)
+			//if err == nil {
+			//	t.Log("key", key, "value", v)
+			//} else {
+			//	t.Error("key", key, err)
+			//}
+			v, err := l.Get(key, fetch)
+			//time.Sleep(time.Second / 100)
+			func(v ...any) {}(v, err)
+		}
+	}
+
+	tmp := make(map[string]struct{})
+
+	var wg sync.WaitGroup
+	for i := 0; i < 10000; i++ {
+		wg.Add(1)
+		key := fmt.Sprintf("key_%d", i%200)
+		tmp[key] = struct{}{}
+		go func() {
+			defer wg.Done()
+			//t.Log(key)
+			fn(key, 10000, func() (string, error) {
+				//time.Sleep(time.Second * 3)
+				//t.Log(time.Now(), "key", key, "fetch")
+				//if rand.Uint32()%5 == 0 {
+				//	return "value_" + key, nil
+				//}
+				//return "", errors.New("rand error")
+				return "value_" + key, nil
+			})
+		}()
+
+		//wg.Add(1)
+		//go func() {
+		//	defer wg.Done()
+		//	for i := 0; i < 10; i++ {
+		//		l.Del(key)
+		//		time.Sleep(time.Second / 3)
+		//	}
+		//}()
+	}
+	wg.Wait()
+	t.Log(len(tmp))
+	t.Log(target.String())
+
+}

pkg/localcache/lru/lru_slot.go

@@ -0,0 +1,51 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lru
+
+func NewSlotLRU[K comparable, V any](slotNum int, hash func(K) uint64, create func() LRU[K, V]) LRU[K, V] {
+	x := &slotLRU[K, V]{
+		n:     uint64(slotNum),
+		slots: make([]LRU[K, V], slotNum),
+		hash:  hash,
+	}
+	for i := 0; i < slotNum; i++ {
+		x.slots[i] = create()
+	}
+	return x
+}
+
+type slotLRU[K comparable, V any] struct {
+	n     uint64
+	slots []LRU[K, V]
+	hash  func(k K) uint64
+}
+
+func (x *slotLRU[K, V]) getIndex(k K) uint64 {
+	return x.hash(k) % x.n
+}
+
+func (x *slotLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) {
+	return x.slots[x.getIndex(key)].Get(key, fetch)
+}
+
+func (x *slotLRU[K, V]) Del(key K) bool {
+	return x.slots[x.getIndex(key)].Del(key)
+}
+
+func (x *slotLRU[K, V]) Stop() {
+	for _, slot := range x.slots {
+		slot.Stop()
+	}
+}

pkg/localcache/option.go

@@ -0,0 +1,136 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package localcache
+
+import (
+	"context"
+	"time"
+
+	"github.com/openimsdk/localcache/lru"
+)
+
+func defaultOption() *option {
+	return &option{
+		localSlotNum:    500,
+		localSlotSize:   20000,
+		linkSlotNum:     500,
+		expirationEvict: false,
+		localSuccessTTL: time.Minute,
+		localFailedTTL:  time.Second * 5,
+		delFn:           make([]func(ctx context.Context, key ...string), 0, 2),
+		target:          emptyTarget{},
+	}
+}
+
+type option struct {
+	localSlotNum  int
+	localSlotSize int
+	linkSlotNum   int
+	// expirationEvict: true means that the cache will be actively cleared when the timer expires,
+	// false means that the cache will be lazily deleted.
+	expirationEvict bool
+	localSuccessTTL time.Duration
+	localFailedTTL  time.Duration
+	delFn           []func(ctx context.Context, key ...string)
+	target          lru.Target
+}
+
+type Option func(o *option)
+
+func WithExpirationEvict() Option {
+	return func(o *option) {
+		o.expirationEvict = true
+	}
+}
+
+func WithLazy() Option {
+	return func(o *option) {
+		o.expirationEvict = false
+	}
+}
+
+func WithLocalDisable() Option {
+	return WithLinkSlotNum(0)
+}
+
+func WithLinkDisable() Option {
+	return WithLinkSlotNum(0)
+}
+
+func WithLinkSlotNum(linkSlotNum int) Option {
+	return func(o *option) {
+		o.linkSlotNum = linkSlotNum
+	}
+}
+
+func WithLocalSlotNum(localSlotNum int) Option {
+	return func(o *option) {
+		o.localSlotNum = localSlotNum
+	}
+}
+
+func WithLocalSlotSize(localSlotSize int) Option {
+	return func(o *option) {
+		o.localSlotSize = localSlotSize
+	}
+}
+
+func WithLocalSuccessTTL(localSuccessTTL time.Duration) Option {
+	if localSuccessTTL < 0 {
+		panic("localSuccessTTL should be greater than 0")
+	}
+	return func(o *option) {
+		o.localSuccessTTL = localSuccessTTL
+	}
+}
+
+func WithLocalFailedTTL(localFailedTTL time.Duration) Option {
+	if localFailedTTL < 0 {
+		panic("localFailedTTL should be greater than 0")
+	}
+	return func(o *option) {
+		o.localFailedTTL = localFailedTTL
+	}
+}
+
+func WithTarget(target lru.Target) Option {
+	if target == nil {
+		panic("target should not be nil")
+	}
+	return func(o *option) {
+		o.target = target
+	}
+}
+
+func WithDeleteKeyBefore(fn func(ctx context.Context, key ...string)) Option {
+	if fn == nil {
+		panic("fn should not be nil")
+	}
+	return func(o *option) {
+		o.delFn = append(o.delFn, fn)
+	}
+}
+
+type emptyTarget struct{}
+
+func (e emptyTarget) IncrGetHit() {}
+
+func (e emptyTarget) IncrGetSuccess() {}
+
+func (e emptyTarget) IncrGetFailed() {}
+
+func (e emptyTarget) IncrDelHit() {}
+
+func (e emptyTarget) IncrDelNotFound() {}

pkg/localcache/tool.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 OpenIM. All rights reserved.
+// Copyright © 2024 OpenIM. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,3 +13,11 @@
 // limitations under the License.
 
 package localcache
+
+func AnyValue[V any](v any, err error) (V, error) {
+	if err != nil {
+		var zero V
+		return zero, err
+	}
+	return v.(V), nil
+}

pkg/rpccache/common.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 OpenIM open source community. All rights reserved.
+// Copyright © 2024 OpenIM. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,14 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package admin
+package rpccache
 
-// ClientConfig Client related configuration items.
-type ClientConfig struct {
-	Key   string `gorm:"column:key;primary_key;type:varchar(255)"`
-	Value string `gorm:"column:value;not null;type:text"`
+func newListMap[V comparable](values []V, err error) (*listMap[V], error) {
+	if err != nil {
+		return nil, err
+	}
+	lm := &listMap[V]{
+		List: values,
+		Map:  make(map[V]struct{}, len(values)),
+	}
+	for _, value := range values {
+		lm.Map[value] = struct{}{}
+	}
+	return lm, nil
 }
 
-func (ClientConfig) TableName() string {
-	return "client_config"
+type listMap[V comparable] struct {
+	List []V
+	Map  map[V]struct{}
 }

pkg/rpccache/conversation.go

@@ -0,0 +1,127 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpccache
+
+import (
+	"context"
+
+	pbconversation "github.com/OpenIMSDK/protocol/conversation"
+	"github.com/OpenIMSDK/tools/errs"
+	"github.com/OpenIMSDK/tools/log"
+	"github.com/openimsdk/localcache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
+	"github.com/redis/go-redis/v9"
+)
+
+func NewConversationLocalCache(client rpcclient.ConversationRpcClient, cli redis.UniversalClient) *ConversationLocalCache {
+	lc := config.Config.LocalCache.Conversation
+	log.ZDebug(context.Background(), "ConversationLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable())
+	x := &ConversationLocalCache{
+		client: client,
+		local: localcache.New[any](
+			localcache.WithLocalSlotNum(lc.SlotNum),
+			localcache.WithLocalSlotSize(lc.SlotSize),
+			localcache.WithLinkSlotNum(lc.SlotNum),
+			localcache.WithLocalSuccessTTL(lc.Success()),
+			localcache.WithLocalFailedTTL(lc.Failed()),
+		),
+	}
+	if lc.Enable() {
+		go subscriberRedisDeleteCache(context.Background(), cli, lc.Topic, x.local.DelLocal)
+	}
+	return x
+}
+
+type ConversationLocalCache struct {
+	client rpcclient.ConversationRpcClient
+	local  localcache.Cache[any]
+}
+
+func (c *ConversationLocalCache) GetConversationIDs(ctx context.Context, ownerUserID string) (val []string, err error) {
+	log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs req", "ownerUserID", ownerUserID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs return", "value", val)
+		} else {
+			log.ZError(ctx, "ConversationLocalCache GetConversationIDs return", err)
+		}
+	}()
+	return localcache.AnyValue[[]string](c.local.Get(ctx, cachekey.GetConversationIDsKey(ownerUserID), func(ctx context.Context) (any, error) {
+		log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs rpc", "ownerUserID", ownerUserID)
+		return c.client.GetConversationIDs(ctx, ownerUserID)
+	}))
+}
+
+func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, conversationID string) (val *pbconversation.Conversation, err error) {
+	log.ZDebug(ctx, "ConversationLocalCache GetConversation req", "userID", userID, "conversationID", conversationID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "ConversationLocalCache GetConversation return", "value", val)
+		} else {
+			log.ZError(ctx, "ConversationLocalCache GetConversation return", err)
+		}
+	}()
+	return localcache.AnyValue[*pbconversation.Conversation](c.local.Get(ctx, cachekey.GetConversationKey(userID, conversationID), func(ctx context.Context) (any, error) {
+		log.ZDebug(ctx, "ConversationLocalCache GetConversation rpc", "userID", userID, "conversationID", conversationID)
+		return c.client.GetConversation(ctx, userID, conversationID)
+	}))
+}
+
+func (c *ConversationLocalCache) GetSingleConversationRecvMsgOpt(ctx context.Context, userID, conversationID string) (int32, error) {
+	conv, err := c.GetConversation(ctx, userID, conversationID)
+	if err != nil {
+		return 0, err
+	}
+	return conv.RecvMsgOpt, nil
+}
+
+func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) {
+	conversations := make([]*pbconversation.Conversation, 0, len(conversationIDs))
+	for _, conversationID := range conversationIDs {
+		conversation, err := c.GetConversation(ctx, ownerUserID, conversationID)
+		if err != nil {
+			if errs.ErrRecordNotFound.Is(err) {
+				continue
+			}
+			return nil, err
+		}
+		conversations = append(conversations, conversation)
+	}
+	return conversations, nil
+}
+
+func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (*listMap[string], error) {
+	return localcache.AnyValue[*listMap[string]](c.local.Get(ctx, cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID), func(ctx context.Context) (any, error) {
+		return newListMap(c.client.GetConversationNotReceiveMessageUserIDs(ctx, conversationID))
+	}))
+}
+
+func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) {
+	res, err := c.getConversationNotReceiveMessageUserIDs(ctx, conversationID)
+	if err != nil {
+		return nil, err
+	}
+	return res.List, nil
+}
+
+func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDMap(ctx context.Context, conversationID string) (map[string]struct{}, error) {
+	res, err := c.getConversationNotReceiveMessageUserIDs(ctx, conversationID)
+	if err != nil {
+		return nil, err
+	}
+	return res.Map, nil
+}

pkg/rpccache/friend.go

@@ -0,0 +1,81 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpccache
+
+import (
+	"context"
+
+	"github.com/OpenIMSDK/tools/log"
+	"github.com/openimsdk/localcache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
+	"github.com/redis/go-redis/v9"
+)
+
+func NewFriendLocalCache(client rpcclient.FriendRpcClient, cli redis.UniversalClient) *FriendLocalCache {
+	lc := config.Config.LocalCache.Friend
+	log.ZDebug(context.Background(), "FriendLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable())
+	x := &FriendLocalCache{
+		client: client,
+		local: localcache.New[any](
+			localcache.WithLocalSlotNum(lc.SlotNum),
+			localcache.WithLocalSlotSize(lc.SlotSize),
+			localcache.WithLinkSlotNum(lc.SlotNum),
+			localcache.WithLocalSuccessTTL(lc.Success()),
+			localcache.WithLocalFailedTTL(lc.Failed()),
+		),
+	}
+	if lc.Enable() {
+		go subscriberRedisDeleteCache(context.Background(), cli, lc.Topic, x.local.DelLocal)
+	}
+	return x
+}
+
+type FriendLocalCache struct {
+	client rpcclient.FriendRpcClient
+	local  localcache.Cache[any]
+}
+
+func (f *FriendLocalCache) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (val bool, err error) {
+	log.ZDebug(ctx, "FriendLocalCache IsFriend req", "possibleFriendUserID", possibleFriendUserID, "userID", userID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "FriendLocalCache IsFriend return", "value", val)
+		} else {
+			log.ZError(ctx, "FriendLocalCache IsFriend return", err)
+		}
+	}()
+	return localcache.AnyValue[bool](f.local.GetLink(ctx, cachekey.GetIsFriendKey(possibleFriendUserID, userID), func(ctx context.Context) (any, error) {
+		log.ZDebug(ctx, "FriendLocalCache IsFriend rpc", "possibleFriendUserID", possibleFriendUserID, "userID", userID)
+		return f.client.IsFriend(ctx, possibleFriendUserID, userID)
+	}, cachekey.GetFriendIDsKey(possibleFriendUserID)))
+}
+
+// IsBlack possibleBlackUserID selfUserID.
+func (f *FriendLocalCache) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (val bool, err error) {
+	log.ZDebug(ctx, "FriendLocalCache IsBlack req", "possibleBlackUserID", possibleBlackUserID, "userID", userID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "FriendLocalCache IsBlack return", "value", val)
+		} else {
+			log.ZError(ctx, "FriendLocalCache IsBlack return", err)
+		}
+	}()
+	return localcache.AnyValue[bool](f.local.GetLink(ctx, cachekey.GetIsBlackIDsKey(possibleBlackUserID, userID), func(ctx context.Context) (any, error) {
+		log.ZDebug(ctx, "FriendLocalCache IsBlack rpc", "possibleBlackUserID", possibleBlackUserID, "userID", userID)
+		return f.client.IsBlack(ctx, possibleBlackUserID, userID)
+	}, cachekey.GetBlackIDsKey(userID)))
+}

pkg/rpccache/group.go

@@ -0,0 +1,158 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpccache
+
+import (
+	"context"
+
+	"github.com/OpenIMSDK/protocol/sdkws"
+	"github.com/OpenIMSDK/tools/errs"
+	"github.com/OpenIMSDK/tools/log"
+	"github.com/openimsdk/localcache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
+	"github.com/redis/go-redis/v9"
+)
+
+func NewGroupLocalCache(client rpcclient.GroupRpcClient, cli redis.UniversalClient) *GroupLocalCache {
+	lc := config.Config.LocalCache.Group
+	log.ZDebug(context.Background(), "GroupLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable())
+	x := &GroupLocalCache{
+		client: client,
+		local: localcache.New[any](
+			localcache.WithLocalSlotNum(lc.SlotNum),
+			localcache.WithLocalSlotSize(lc.SlotSize),
+			localcache.WithLinkSlotNum(lc.SlotNum),
+			localcache.WithLocalSuccessTTL(lc.Success()),
+			localcache.WithLocalFailedTTL(lc.Failed()),
+		),
+	}
+	if lc.Enable() {
+		go subscriberRedisDeleteCache(context.Background(), cli, lc.Topic, x.local.DelLocal)
+	}
+	return x
+}
+
+type GroupLocalCache struct {
+	client rpcclient.GroupRpcClient
+	local  localcache.Cache[any]
+}
+
+func (g *GroupLocalCache) getGroupMemberIDs(ctx context.Context, groupID string) (val *listMap[string], err error) {
+	log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs req", "groupID", groupID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs return", "value", val)
+		} else {
+			log.ZError(ctx, "GroupLocalCache getGroupMemberIDs return", err)
+		}
+	}()
+	return localcache.AnyValue[*listMap[string]](g.local.Get(ctx, cachekey.GetGroupMemberIDsKey(groupID), func(ctx context.Context) (any, error) {
+		log.ZDebug(ctx, "GroupLocalCache getGroupMemberIDs rpc", "groupID", groupID)
+		return newListMap(g.client.GetGroupMemberIDs(ctx, groupID))
+	}))
+}
+
+func (g *GroupLocalCache) GetGroupMember(ctx context.Context, groupID, userID string) (val *sdkws.GroupMemberFullInfo, err error) {
+	log.ZDebug(ctx, "GroupLocalCache GetGroupInfo req", "groupID", groupID, "userID", userID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "value", val)
+		} else {
+			log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err)
+		}
+	}()
+	return localcache.AnyValue[*sdkws.GroupMemberFullInfo](g.local.Get(ctx, cachekey.GetGroupMemberInfoKey(groupID, userID), func(ctx context.Context) (any, error) {
+		log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID, "userID", userID)
+		return g.client.GetGroupMemberCache(ctx, groupID, userID)
+	}))
+}
+
+func (g *GroupLocalCache) GetGroupInfo(ctx context.Context, groupID string) (val *sdkws.GroupInfo, err error) {
+	log.ZDebug(ctx, "GroupLocalCache GetGroupInfo req", "groupID", groupID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "GroupLocalCache GetGroupInfo return", "value", val)
+		} else {
+			log.ZError(ctx, "GroupLocalCache GetGroupInfo return", err)
+		}
+	}()
+	return localcache.AnyValue[*sdkws.GroupInfo](g.local.Get(ctx, cachekey.GetGroupInfoKey(groupID), func(ctx context.Context) (any, error) {
+		log.ZDebug(ctx, "GroupLocalCache GetGroupInfo rpc", "groupID", groupID)
+		return g.client.GetGroupInfoCache(ctx, groupID)
+	}))
+}
+
+func (g *GroupLocalCache) GetGroupMemberIDs(ctx context.Context, groupID string) ([]string, error) {
+	res, err := g.getGroupMemberIDs(ctx, groupID)
+	if err != nil {
+		return nil, err
+	}
+	return res.List, nil
+}
+
+func (g *GroupLocalCache) GetGroupMemberIDMap(ctx context.Context, groupID string) (map[string]struct{}, error) {
+	res, err := g.getGroupMemberIDs(ctx, groupID)
+	if err != nil {
+		return nil, err
+	}
+	return res.Map, nil
+}
+
+func (g *GroupLocalCache) GetGroupInfos(ctx context.Context, groupIDs []string) ([]*sdkws.GroupInfo, error) {
+	groupInfos := make([]*sdkws.GroupInfo, 0, len(groupIDs))
+	for _, groupID := range groupIDs {
+		groupInfo, err := g.GetGroupInfo(ctx, groupID)
+		if err != nil {
+			if errs.ErrRecordNotFound.Is(err) {
+				continue
+			}
+			return nil, err
+		}
+		groupInfos = append(groupInfos, groupInfo)
+	}
+	return groupInfos, nil
+}
+
+func (g *GroupLocalCache) GetGroupMembers(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) {
+	members := make([]*sdkws.GroupMemberFullInfo, 0, len(userIDs))
+	for _, userID := range userIDs {
+		member, err := g.GetGroupMember(ctx, groupID, userID)
+		if err != nil {
+			if errs.ErrRecordNotFound.Is(err) {
+				continue
+			}
+			return nil, err
+		}
+		members = append(members, member)
+	}
+	return members, nil
+}
+
+func (g *GroupLocalCache) GetGroupMemberInfoMap(ctx context.Context, groupID string, userIDs []string) (map[string]*sdkws.GroupMemberFullInfo, error) {
+	members := make(map[string]*sdkws.GroupMemberFullInfo)
+	for _, userID := range userIDs {
+		member, err := g.GetGroupMember(ctx, groupID, userID)
+		if err != nil {
+			if errs.ErrRecordNotFound.Is(err) {
+				continue
+			}
+			return nil, err
+		}
+		members[userID] = member
+	}
+	return members, nil
+}

pkg/rpccache/subscriber.go

@@ -0,0 +1,38 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpccache
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/OpenIMSDK/tools/log"
+	"github.com/redis/go-redis/v9"
+)
+
+func subscriberRedisDeleteCache(ctx context.Context, client redis.UniversalClient, channel string, del func(ctx context.Context, key ...string)) {
+	for message := range client.Subscribe(ctx, channel).Channel() {
+		log.ZDebug(ctx, "subscriberRedisDeleteCache", "channel", channel, "payload", message.Payload)
+		var keys []string
+		if err := json.Unmarshal([]byte(message.Payload), &keys); err != nil {
+			log.ZError(ctx, "subscriberRedisDeleteCache json.Unmarshal error", err)
+			continue
+		}
+		if len(keys) == 0 {
+			continue
+		}
+		del(ctx, keys...)
+	}
+}

pkg/rpccache/user.go

@@ -0,0 +1,112 @@
+// Copyright © 2024 OpenIM. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rpccache
+
+import (
+	"context"
+
+	"github.com/OpenIMSDK/protocol/sdkws"
+	"github.com/OpenIMSDK/tools/errs"
+	"github.com/OpenIMSDK/tools/log"
+	"github.com/openimsdk/localcache"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
+	"github.com/openimsdk/open-im-server/v3/pkg/rpcclient"
+	"github.com/redis/go-redis/v9"
+)
+
+func NewUserLocalCache(client rpcclient.UserRpcClient, cli redis.UniversalClient) *UserLocalCache {
+	lc := config.Config.LocalCache.User
+	log.ZDebug(context.Background(), "UserLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable())
+	x := &UserLocalCache{
+		client: client,
+		local: localcache.New[any](
+			localcache.WithLocalSlotNum(lc.SlotNum),
+			localcache.WithLocalSlotSize(lc.SlotSize),
+			localcache.WithLinkSlotNum(lc.SlotNum),
+			localcache.WithLocalSuccessTTL(lc.Success()),
+			localcache.WithLocalFailedTTL(lc.Failed()),
+		),
+	}
+	if lc.Enable() {
+		go subscriberRedisDeleteCache(context.Background(), cli, lc.Topic, x.local.DelLocal)
+	}
+	return x
+}
+
+type UserLocalCache struct {
+	client rpcclient.UserRpcClient
+	local  localcache.Cache[any]
+}
+
+func (u *UserLocalCache) GetUserInfo(ctx context.Context, userID string) (val *sdkws.UserInfo, err error) {
+	log.ZDebug(ctx, "UserLocalCache GetUserInfo req", "userID", userID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "UserLocalCache GetUserInfo return", "value", val)
+		} else {
+			log.ZError(ctx, "UserLocalCache GetUserInfo return", err)
+		}
+	}()
+	return localcache.AnyValue[*sdkws.UserInfo](u.local.Get(ctx, cachekey.GetUserInfoKey(userID), func(ctx context.Context) (any, error) {
+		log.ZDebug(ctx, "UserLocalCache GetUserInfo rpc", "userID", userID)
+		return u.client.GetUserInfo(ctx, userID)
+	}))
+}
+
+func (u *UserLocalCache) GetUserGlobalMsgRecvOpt(ctx context.Context, userID string) (val int32, err error) {
+	log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt req", "userID", userID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt return", "value", val)
+		} else {
+			log.ZError(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt return", err)
+		}
+	}()
+	return localcache.AnyValue[int32](u.local.Get(ctx, cachekey.GetUserGlobalRecvMsgOptKey(userID), func(ctx context.Context) (any, error) {
+		log.ZDebug(ctx, "UserLocalCache GetUserGlobalMsgRecvOpt rpc", "userID", userID)
+		return u.client.GetUserGlobalMsgRecvOpt(ctx, userID)
+	}))
+}
+
+func (u *UserLocalCache) GetUsersInfo(ctx context.Context, userIDs []string) ([]*sdkws.UserInfo, error) {
+	users := make([]*sdkws.UserInfo, 0, len(userIDs))
+	for _, userID := range userIDs {
+		user, err := u.GetUserInfo(ctx, userID)
+		if err != nil {
+			if errs.ErrRecordNotFound.Is(err) {
+				continue
+			}
+			return nil, err
+		}
+		users = append(users, user)
+	}
+	return users, nil
+}
+
+func (u *UserLocalCache) GetUsersInfoMap(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error) {
+	users := make(map[string]*sdkws.UserInfo, len(userIDs))
+	for _, userID := range userIDs {
+		user, err := u.GetUserInfo(ctx, userID)
+		if err != nil {
+			if errs.ErrRecordNotFound.Is(err) {
+				continue
+			}
+			return nil, err
+		}
+		users[userID] = user
+	}
+	return users, nil
+}

pkg/rpcclient/conversation.go

@@ -114,6 +114,14 @@ func (c *ConversationRpcClient) GetConversationsByConversationID(ctx context.Con
 	return resp.Conversations, nil
 }
 
+func (c *ConversationRpcClient) GetConversationOfflinePushUserIDs(ctx context.Context, conversationID string, userIDs []string) ([]string, error) {
+	resp, err := c.Client.GetConversationOfflinePushUserIDs(ctx, &pbconversation.GetConversationOfflinePushUserIDsReq{ConversationID: conversationID, UserIDs: userIDs})
+	if err != nil {
+		return nil, err
+	}
+	return resp.UserIDs, nil
+}
+
 func (c *ConversationRpcClient) GetConversations(ctx context.Context, ownerUserID string, conversationIDs []string) ([]*pbconversation.Conversation, error) {
 	if len(conversationIDs) == 0 {
 		return nil, nil
@@ -127,3 +135,11 @@ func (c *ConversationRpcClient) GetConversations(ctx context.Context, ownerUserI
 	}
 	return resp.Conversations, nil
 }
+
+func (c *ConversationRpcClient) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) {
+	resp, err := c.Client.GetConversationNotReceiveMessageUserIDs(ctx, &pbconversation.GetConversationNotReceiveMessageUserIDsReq{ConversationID: conversationID})
+	if err != nil {
+		return nil, err
+	}
+	return resp.UserIDs, nil
+}

pkg/rpcclient/friend.go

@@ -80,7 +80,7 @@ func (f *FriendRpcClient) GetFriendIDs(ctx context.Context, ownerUserID string)
 	return resp.FriendIDs, nil
 }
 
-func (b *FriendRpcClient) IsBlocked(ctx context.Context, possibleBlackUserID, userID string) (bool, error) {
+func (b *FriendRpcClient) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (bool, error) {
 	r, err := b.Client.IsBlack(ctx, &friend.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID})
 	if err != nil {
 		return false, err

pkg/rpcclient/group.go

@@ -26,11 +26,9 @@ import (
 	"github.com/OpenIMSDK/tools/utils"
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	util "github.com/openimsdk/open-im-server/v3/pkg/util/genutil"
-	"google.golang.org/grpc"
 )
 
 type Group struct {
-	conn   grpc.ClientConnInterface
 	Client group.GroupClient
 	discov discoveryregistry.SvcDiscoveryRegistry
 	Config *config.GlobalConfig
@@ -42,7 +40,7 @@ func NewGroup(discov discoveryregistry.SvcDiscoveryRegistry, config *config.Glob
 		util.ExitWithError(err)
 	}
 	client := group.NewGroupClient(conn)
-	return &Group{discov: discov, conn: conn, Client: client, Config: config}
+	return &Group{discov: discov, Client: client, Config: config}
 }
 
 type GroupRpcClient Group

scripts/cherry-pick.sh

@@ -21,10 +21,6 @@
 # checks them out to a branch named:
 #  automated-cherry-pick-of-<pr>-<target branch>-<timestamp>
 
-
-
-
-
 OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${OPENIM_ROOT}/scripts/lib/init.sh"
 

scripts/docker-start-all.sh

@@ -25,5 +25,5 @@ OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${OPENIM_ROOT}/scripts/install/common.sh"
 openim::log::info "\n# Use Docker to start all openim service"
 trap 'openim::util::onCtrlC' INT
-"${OPENIM_ROOT}"/scripts/start-all.sh
+"${OPENIM_ROOT}"/scripts/start-all.sh 2>&1 &
 tail -f ${DOCKER_LOG_FILE}

scripts/init-config.sh

@@ -150,6 +150,12 @@ process_file() {
       exit 1
     fi
     if [[ -n "${env_cmd}" ]]; then
+
+    {
+        printf "debugggggggggggggggggggg file: %s template: %s\n" "${ENV_FILE}" "${template}"
+    } | tee /tmp/debug.log
+
+
       eval "$env_cmd ${OPENIM_ROOT}/scripts/genconfig.sh '${ENV_FILE}' '${template}' > '${output_file}'" || {
         openim::log::error "Error processing template file ${template}"
         exit 1

scripts/install/openim-tools.sh

@@ -58,7 +58,6 @@ openim::tools::pre_start_name() {
   local targets=(
     ncpu
     component
-    up35
   )
   echo "${targets[@]}"
 }
@@ -101,7 +100,13 @@ function openim::tools::start_service() {
     cmd="${cmd} --prometheus_port ${prometheus_port}"
   fi
   openim::log::status "Starting binary ${binary_name}..."
+
+
+
  ${cmd} 
+
+
+
  local status=$?
 
     if [ $status -eq 0 ]; then

scripts/install/test.sh

@@ -1272,7 +1272,7 @@ openim::test::search_msg() {
 {
   "sendID": "${sendID}",
   "recvID": "${recvID}",
-  "msgType": ${msgType},
+  "contentType": ${msgType},
   "sendTime": "${sendTime}",
   "sessionType": ${sessionType},
   "pagination": {

scripts/lib/util.sh

@@ -1795,10 +1795,6 @@ openim::util::stop_services_on_ports() {
         return 0
     fi
 }
-# nc -l -p 12345
-# nc -l -p 123456
-# ps -ef | grep "nc -l"
-# openim::util::stop_services_on_ports 1234 12345
 
 
 # The `openim::util::stop_services_with_name` function stops services with specified names.
@@ -1808,68 +1804,7 @@ openim::util::stop_services_on_ports() {
 # Usage:
 # openim::util::stop_services_with_name nginx apache
 # The function returns a status of 1 if any service couldn't be stopped.
-openim::util::stop_services_with_name() {
-    # An array to collect names of processes that couldn't be stopped.
-    local not_stopped=()
 
-    # An array to collect information about processes that were stopped.
-    local stopped=()
-
-    echo "Stopping services with names: $*"
-    # Iterate over each given service name.
-    for server_name in "$@"; do
-        # Use the `pgrep` command to find process IDs related to the given service name.
-        local pids=$(pgrep -f "$server_name")
-
-        # If no process was found with the name, add it to the not_stopped list
-        if [[ -z $pids ]]; then
-            not_stopped+=("$server_name")
-            continue
-        fi
-        local stopped_this_time=false
-        for pid in $pids; do
-
-            # Exclude the PID of the current script
-            if [[ "$pid" == "$$" ]]; then
-                continue
-            fi
-
-            # If there's a Process ID, it means the service with the name is running.
-            if [[ -n $pid ]]; then
-                # Try to stop the service by killing its process.
-                if kill -10 $pid 2>/dev/null; then
-                    stopped_this_time=true
-                fi
-            fi
-        done
-
-        if $stopped_this_time; then
-            stopped+=("$server_name")
-        else
-            not_stopped+=("$server_name")
-        fi
-    done
-
-    # Print information about services whose processes couldn't be stopped.
-    if [[ ${#not_stopped[@]} -ne 0 ]]; then
-        echo "Services that couldn't be stopped:"
-        for name in "${not_stopped[@]}"; do
-            openim::log::status "Failed to stop the $name service."
-        done
-    fi
-
-    # Print information about services whose processes were successfully stopped.
-    if [[ ${#stopped[@]} -ne 0 ]]; then
-        echo
-        echo "Stopped services:"
-        for name in "${stopped[@]}"; do
-            echo "Successfully stopped the $name service."
-        done
-    fi
-
-    openim::log::success "All specified services were stopped."
-    echo ""
-}
 # sleep 333333&
 # sleep 444444&
 # ps -ef | grep "sleep"

scripts/start-all.sh

@@ -63,7 +63,7 @@ function execute_start_scripts() {
 
 
 if openim::util::is_running_in_container; then
-  exec > ${DOCKER_LOG_FILE} 2>&1
+  exec >> ${DOCKER_LOG_FILE} 2>&1
 fi
 
 

scripts/verify-shellcheck.sh

@@ -87,6 +87,7 @@ disabled=(
   2178
   2064
   2260
+  2261
   2043
   2178
   2044

tools/component/component.go

@@ -30,9 +30,9 @@ import (
 	"github.com/OpenIMSDK/tools/component"
 	"github.com/OpenIMSDK/tools/errs"
 
-	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
-
 	"gopkg.in/yaml.v3"
+
+	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 )
 
 const (
@@ -66,6 +66,23 @@ type checkFunc struct {
 	config   *config.GlobalConfig
 }
 
+// colorErrPrint prints formatted string in red to stderr
+func colorErrPrint(msg string) {
+	// ANSI escape code for red text
+	const redColor = "\033[31m"
+	// ANSI escape code to reset color
+	const resetColor = "\033[0m"
+	msg = redColor + msg + resetColor
+	// Print to stderr in red
+	fmt.Fprintf(os.Stderr, "%s\n", msg)
+}
+
+func colorSuccessPrint(format string, a ...interface{}) {
+	// ANSI escape code for green text is \033[32m
+	// \033[0m resets the color
+	fmt.Printf("\033[32m"+format+"\033[0m", a...)
+}
+
 func main() {
 	flag.Parse()
 
@@ -77,7 +94,7 @@ func main() {
 
 	err = configGetEnv(conf)
 	if err != nil {
-		fmt.Printf("configGetEnv failed,err:%v", err)
+		fmt.Printf("configGetEnv failed, err:%v", err)
 		return
 	}
 
@@ -94,7 +111,7 @@ func main() {
 		if i != 0 {
 			time.Sleep(1 * time.Second)
 		}
-		fmt.Printf("Checking components Round %v...\n", i+1)
+		fmt.Printf("Checking components round %v...\n", i+1)
 
 		var err error
 		allSuccess := true
@@ -102,33 +119,31 @@ func main() {
 			if !check.flag {
 				err = check.function(check.config)
 				if err != nil {
-					if errors.Is(err, errMinioNotEnabled) {
-						fmt.Println(err.Error())
+					allSuccess = false
+					colorErrPrint(fmt.Sprintf("Check component: %s, failed: %v", check.name, err.Error()))
+
+					if check.name == "Minio" {
+						if errors.Is(err, errMinioNotEnabled) ||
+							errors.Is(err, errSignEndPoint) ||
+							errors.Is(err, errApiURL) {
 							checks[index].flag = true
+							continue
 						}
-					if errors.Is(err, errSignEndPoint) {
-						fmt.Fprintf(os.Stderr, err.Error())
-						checks[index].flag = true
-					}
-					component.ErrorPrint(fmt.Sprintf("Starting %s failed:%v.", check.name, errs.Unwrap(err).Error()))
-					if !strings.Contains(errs.Unwrap(err).Error(), "connection refused") &&
-						!strings.Contains(errs.Unwrap(err).Error(), "timeout waiting") {
-						component.ErrorPrint("Some components started failed!")
-						os.Exit(-1)
+						break
 					}
 				} else {
 					checks[index].flag = true
 					component.SuccessPrint(fmt.Sprintf("%s connected successfully", check.name))
 				}
 			}
-		}
 
+		}
 		if allSuccess {
 			component.SuccessPrint("All components started successfully!")
 			return
 		}
 	}
-	component.ErrorPrint("Some components started failed!")
+	component.ErrorPrint("Some components checked failed!")
 	os.Exit(-1)
 }
 
@@ -166,13 +181,13 @@ func checkRedis(config *config.GlobalConfig) error {
 // checkMinio checks the MinIO connection
 func checkMinio(config *config.GlobalConfig) error {
 	if strings.Contains(config.Object.ApiURL, "127.0.0.1") {
-		return errs.Wrap(errApiURL, "config.Object.ApiURL: "+config.Object.ApiURL)
+		return errs.Wrap(errApiURL)
 	}
 	if config.Object.Enable != "minio" {
-		return errs.Wrap(errMinioNotEnabled, "config.Object.Enable: "+config.Object.Enable)
+		return errs.Wrap(errMinioNotEnabled)
 	}
 	if strings.Contains(config.Object.Minio.Endpoint, "127.0.0.1") {
-		return errs.Wrap(errSignEndPoint, "config.Object.Minio.Endpoint: "+config.Object.Minio.Endpoint)
+		return errs.Wrap(errSignEndPoint)
 	}
 
 	minio := &component.Minio{
@@ -258,13 +273,13 @@ func checkKafka(config *config.GlobalConfig) error {
 	_, err = kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{
 		KafkaVersion:   sarama.V2_0_0_0,
 		OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false,
-	}, []string{config.Kafka.MsgToPush.Topic},
+	}, []string{config.Kafka.MsgToMongo.Topic},
 		config.Kafka.Addr, config.Kafka.ConsumerGroupID.MsgToMongo, tlsConfig)
 	if err != nil {
 		return err
 	}
 
-	kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{
+	_, err = kafka.NewMConsumerGroup(&kafka.MConsumerGroupConfig{
 		KafkaVersion:   sarama.V2_0_0_0,
 		OffsetsInitial: sarama.OffsetNewest, IsReturnErr: false,
 	}, []string{config.Kafka.MsgToPush.Topic}, config.Kafka.Addr,
@@ -292,15 +307,15 @@ func configGetEnv(config *config.GlobalConfig) error {
 	config.Mongo.Password = getEnv("MONGO_OPENIM_PASSWORD", config.Mongo.Password)
 	config.Mongo.Address = getArrEnv("MONGO_ADDRESS", "MONGO_PORT", config.Mongo.Address)
 	config.Mongo.Database = getEnv("MONGO_DATABASE", config.Mongo.Database)
-	maxPoolSize, err := getEnvInt("MONGO_DATABASE", config.Mongo.MaxPoolSize)
+	maxPoolSize, err := getEnvInt("MONGO_MAX_POOL_SIZE", config.Mongo.MaxPoolSize)
 	if err != nil {
-		return err
+		return errs.Wrap(err, "MONGO_MAX_POOL_SIZE")
 	}
 	config.Mongo.MaxPoolSize = maxPoolSize
 
 	config.Redis.Username = getEnv("REDIS_USERNAME", config.Redis.Username)
 	config.Redis.Password = getEnv("REDIS_PASSWORD", config.Redis.Password)
-	config.Redis.Address = getArrEnv("REDIS_ADDRESS", "REDIS_PASSWORD", config.Redis.Address)
+	config.Redis.Address = getArrEnv("REDIS_ADDRESS", "REDIS_PORT", config.Redis.Address)
 
 	config.Object.ApiURL = getEnv("OBJECT_APIURL", config.Object.ApiURL)
 	config.Object.Minio.Endpoint = getEnv("MINIO_ENDPOINT", config.Object.Minio.Endpoint)

tools/data-conversion/README.md

@@ -1,98 +0,0 @@
-# OpenIM V2 至 V3 数据迁移指南
-
-该指南提供了从 OpenIM V2 迁移至 V3 的详细步骤。请确保在开始迁移过程之前，熟悉所有步骤，并按照指南准确执行。
-
-+ [OpenIM Chat](https://github.com/OpenIMSDK/chat)
-+ [OpenIM Server](https://github.com/OpenIMSDK/Open-IM-Server)
-
-
-
-### 1. 数据备份
-
-在开始数据迁移之前，强烈建议备份所有相关的数据以防止任何可能的数据丢失。
-
-### 2. 迁移 OpenIM MySQL 数据
-
-+ 位置: `open-im-server/tools/data-conversion/openim/cmd/conversion-mysql.go`
-+ 配置 `conversion-mysql.go` 文件中的数据库信息。
-+ 手动创建 V3 版本的数据库，并确保字符集为 `utf8mb4`。
-
-```bash
-// V2 数据库配置
-var (
-    usernameV2 = "root"
-    passwordV2 = "openIM"
-    addrV2     = "127.0.0.1:13306"
-    databaseV2 = "openIM_v2"
-)
-
-// V3 数据库配置
-var (
-    usernameV3 = "root"
-    passwordV3 = "openIM123"
-    addrV3     = "127.0.0.1:13306"
-    databaseV3 = "openim_v3"
-)
-```
-
-**执行数据迁移命令：**
-
-```bash
-make build BINS="conversion-mysql"
-```
-
-启动的二进制在 `_output/bin/tools` 中
-
-
-### 3. 转换聊天消息（可选）
-
-+ 只支持转换存储在 Kafka 中的消息。
-+ 位置: `open-im-server/tools/data-conversion/openim/conversion-msg/conversion-msg.go`
-+ 配置 `msg.go` 文件中的消息和服务器信息。
-
-```bash
-var (
-	topic       = "ws2ms_chat"      // V2 版本 Kafka 主题
-	kafkaAddr   = "127.0.0.1:9092"  // V2 版本 Kafka 地址
-	rpcAddr     = "127.0.0.1:10130" // V3 版本 RPC 地址
-	adminUserID = "openIM123456"    // V3 版本管理员用户ID
-	concurrency = 4                 // 并发数量
-)
-```
-
-**执行数据迁移命令：**
-
-```bash
-make build BINS="conversion-msg"
-```
-
-### 4. 转换业务服务器数据
-
-+ 只支持转换存储在 Kafka 中的消息。
-+ 位置: `open-im-server/tools/data-conversion/chat/cmd/conversion-chat/chat.go`
-+ 需要手动创建 V3 版本的数据库，并确保字符集为 `utf8mb4`。
-+ 配置 `main.go` 文件中的数据库信息。
-
-```bash
-// V2 数据库配置
-var (
-	usernameV2 = "root"
-	passwordV2 = "openIM"
-	addrV2     = "127.0.0.1:13306"
-	databaseV2 = "admin_chat"
-)
-
-// V3 数据库配置
-var (
-	usernameV3 = "root"
-	passwordV3 = "openIM123"
-	addrV3     = "127.0.0.1:13306"
-	databaseV3 = "openim_enterprise"
-)
-```
-
-**执行数据迁移命令：**
-
-```bash
-make build BINS="conversion-chat"
-```

tools/data-conversion/chat/cmd/conversion-chat/chat.go

@@ -1,77 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"fmt"
-	"log"
-
-	"gorm.io/driver/mysql"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
-
-	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/conversion"
-	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/utils"
-)
-
-func main() {
-	var (
-		usernameV2 = "root"            // Username for MySQL v2 version
-		passwordV2 = "openIM"          // Password for MySQL v2 version
-		addrV2     = "127.0.0.1:13306" // Address for MySQL v2 version
-		databaseV2 = "admin_chat"      // Database name for MySQL v2 version
-	)
-
-	var (
-		usernameV3 = "root"              // Username for MySQL v3 version
-		passwordV3 = "openIM123"         // Password for MySQL v3 version
-		addrV3     = "127.0.0.1:13306"   // Address for MySQL v3 version
-		databaseV3 = "openim_enterprise" // Database name for MySQL v3 version
-	)
-
-	var concurrency = 1 // Concurrency quantity
-
-	log.SetFlags(log.LstdFlags | log.Llongfile)
-	dsnV2 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV2, passwordV2, addrV2, databaseV2)
-	dsnV3 := fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", usernameV3, passwordV3, addrV3, databaseV3)
-	dbV2, err := gorm.Open(mysql.Open(dsnV2), &gorm.Config{Logger: logger.Discard})
-	if err != nil {
-		log.Println("open v2 db failed", err)
-		return
-	}
-	dbV3, err := gorm.Open(mysql.Open(dsnV3), &gorm.Config{Logger: logger.Discard})
-	if err != nil {
-		log.Println("open v3 db failed", err)
-		return
-	}
-
-	var tasks utils.TakeList
-
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Account) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Attribute) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Register) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.UserLoginRecord) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Admin) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.Applet) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.ForbiddenAccount) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.InvitationRegister) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.IPForbidden) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.LimitUserLoginIP) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.RegisterAddFriend) })
-	tasks.Append(func() (string, error) { return utils.FindAndInsert(dbV2, dbV3, conversion.RegisterAddGroup) })
-
-	utils.RunTask(concurrency, tasks)
-
-}

tools/data-conversion/chat/conversion/conversion.go

@@ -1,167 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package conversion
-
-import (
-	v2 "github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/v2"
-	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/v3/admin"
-	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/chat/v3/chat"
-	"github.com/openimsdk/open-im-server/v3/tools/data-conversion/utils"
-)
-
-// ########## chat ##########
-
-func Account(v v2.Account) (chat.Account, bool) {
-	utils.InitTime(&v.CreateTime, &v.ChangeTime)
-	return chat.Account{
-		UserID:         v.UserID,
-		Password:       v.Password,
-		CreateTime:     v.CreateTime,
-		ChangeTime:     v.ChangeTime,
-		OperatorUserID: v.OperatorUserID,
-	}, true
-}
-
-func Attribute(v v2.Attribute) (chat.Attribute, bool) {
-	utils.InitTime(&v.CreateTime, &v.ChangeTime, &v.BirthTime)
-	return chat.Attribute{
-		UserID:           v.UserID,
-		Account:          v.Account,
-		PhoneNumber:      v.PhoneNumber,
-		AreaCode:         v.AreaCode,
-		Email:            v.Email,
-		Nickname:         v.Nickname,
-		FaceURL:          v.FaceURL,
-		Gender:           v.Gender,
-		CreateTime:       v.CreateTime,
-		ChangeTime:       v.ChangeTime,
-		BirthTime:        v.BirthTime,
-		Level:            v.Level,
-		AllowVibration:   v.AllowVibration,
-		AllowBeep:        v.AllowBeep,
-		AllowAddFriend:   v.AllowAddFriend,
-		GlobalRecvMsgOpt: 0,
-	}, true
-}
-
-func Register(v v2.Register) (chat.Register, bool) {
-	utils.InitTime(&v.CreateTime)
-	return chat.Register{
-		UserID:      v.UserID,
-		DeviceID:    v.DeviceID,
-		IP:          v.IP,
-		Platform:    v.Platform,
-		AccountType: v.AccountType,
-		Mode:        v.Mode,
-		CreateTime:  v.CreateTime,
-	}, true
-}
-
-func UserLoginRecord(v v2.UserLoginRecord) (chat.UserLoginRecord, bool) {
-	utils.InitTime(&v.LoginTime)
-	return chat.UserLoginRecord{
-		UserID:    v.UserID,
-		LoginTime: v.LoginTime,
-		IP:        v.IP,
-		DeviceID:  v.DeviceID,
-		Platform:  v.Platform,
-	}, true
-}
-
-// ########## admin ##########
-
-func Admin(v v2.Admin) (admin.Admin, bool) {
-	utils.InitTime(&v.CreateTime)
-	return admin.Admin{
-		Account:    v.Account,
-		Password:   v.Password,
-		FaceURL:    v.FaceURL,
-		Nickname:   v.Nickname,
-		UserID:     v.UserID,
-		Level:      v.Level,
-		CreateTime: v.CreateTime,
-	}, true
-}
-
-func Applet(v v2.Applet) (admin.Applet, bool) {
-	utils.InitTime(&v.CreateTime)
-	return admin.Applet{
-		ID:         v.ID,
-		Name:       v.Name,
-		AppID:      v.AppID,
-		Icon:       v.Icon,
-		URL:        v.URL,
-		MD5:        v.MD5,
-		Size:       v.Size,
-		Version:    v.Version,
-		Priority:   v.Priority,
-		Status:     v.Status,
-		CreateTime: v.CreateTime,
-	}, true
-}
-
-func ForbiddenAccount(v v2.ForbiddenAccount) (admin.ForbiddenAccount, bool) {
-	utils.InitTime(&v.CreateTime)
-	return admin.ForbiddenAccount{
-		UserID:         v.UserID,
-		Reason:         v.Reason,
-		OperatorUserID: v.OperatorUserID,
-		CreateTime:     v.CreateTime,
-	}, true
-}
-
-func InvitationRegister(v v2.InvitationRegister) (admin.InvitationRegister, bool) {
-	utils.InitTime(&v.CreateTime)
-	return admin.InvitationRegister{
-		InvitationCode: v.InvitationCode,
-		UsedByUserID:   v.UsedByUserID,
-		CreateTime:     v.CreateTime,
-	}, true
-}
-
-func IPForbidden(v v2.IPForbidden) (admin.IPForbidden, bool) {
-	utils.InitTime(&v.CreateTime)
-	return admin.IPForbidden{
-		IP:            v.IP,
-		LimitRegister: v.LimitRegister > 0,
-		LimitLogin:    v.LimitLogin > 0,
-		CreateTime:    v.CreateTime,
-	}, true
-}
-
-func LimitUserLoginIP(v v2.LimitUserLoginIP) (admin.LimitUserLoginIP, bool) {
-	utils.InitTime(&v.CreateTime)
-	return admin.LimitUserLoginIP{
-		UserID:     v.UserID,
-		IP:         v.IP,
-		CreateTime: v.CreateTime,
-	}, true
-}
-
-func RegisterAddFriend(v v2.RegisterAddFriend) (admin.RegisterAddFriend, bool) {
-	utils.InitTime(&v.CreateTime)
-	return admin.RegisterAddFriend{
-		UserID:     v.UserID,
-		CreateTime: v.CreateTime,
-	}, true
-}
-
-func RegisterAddGroup(v v2.RegisterAddGroup) (admin.RegisterAddGroup, bool) {
-	utils.InitTime(&v.CreateTime)
-	return admin.RegisterAddGroup{
-		GroupID:    v.GroupID,
-		CreateTime: v.CreateTime,
-	}, true
-}

tools/data-conversion/chat/v2/admin.go

@@ -1,77 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package v2
-
-import (
-	"time"
-)
-
-// AppVersion manages PC client versions
-type AppVersion struct {
-	Version     string `gorm:"column:version;size:64"  json:"version"`
-	Type        int    `gorm:"column:type;primary_key" json:"type"`
-	UpdateTime  int    `gorm:"column:update_time"      json:"update_time"`
-	ForceUpdate bool   `gorm:"column:force_update"     json:"force_update"`
-	FileName    string `gorm:"column:file_name"        json:"file_name"`
-	YamlName    string `gorm:"column:yaml_name"        json:"yaml_name"`
-	UpdateLog   string `gorm:"column:update_log"       json:"update_log"`
-}
-
-// Admin manages backend administrators
-type Admin struct {
-	Account    string    `gorm:"column:account;primary_key;type:char(64)" json:"account"`
-	Password   string    `gorm:"column:Password;type:char(64)"            json:"password"`
-	FaceURL    string    `gorm:"column:FaceURL;type:char(64)"             json:"faceURL"`
-	Nickname   string    `gorm:"column:Nickname;type:char(64)"            json:"nickname"`
-	UserID     string    `gorm:"column:UserID;type:char(64)"              json:"userID"` //openIM userID
-	Level      int32     `gorm:"column:level;default:1"                   json:"level"`
-	CreateTime time.Time `gorm:"column:create_time"                       json:"createTime"`
-}
-
-// RegisterAddFriend specifies default friends when registering
-type RegisterAddFriend struct {
-	UserID     string    `gorm:"column:user_id;primary_key;type:char(64)" json:"userID"`
-	CreateTime time.Time `gorm:"column:create_time"                       json:"createTime"`
-}
-
-// RegisterAddGroup specifies default groups when registering
-type RegisterAddGroup struct {
-	GroupID    string    `gorm:"column:group_id;primary_key;type:char(64)" json:"userID"`
-	CreateTime time.Time `gorm:"column:create_time"                        json:"createTime"`
-}
-
-// ClientInitConfig contains system-related configuration items
-type ClientInitConfig struct {
-	DiscoverPageURL            string `gorm:"column:discover_page_url;size:128"              json:"discoverPageURL"`
-	OrdinaryUserAddFriend      int32  `gorm:"column:ordinary_user_add_friend; default:1"     json:"ordinaryUserAddFriend"`
-	BossUserID                 string `gorm:"column:boss_user_id;type:char(64)"              json:"bossUserID"`
-	AdminURL                   string `gorm:"column:admin_url;type:char(128)"                json:"adminURL"`
-	AllowSendMsgNotFriend      int32  `gorm:"column:allow_send_msg_not_friend;default:1"     json:"allowSendMsgNotFriend"`
-	NeedInvitationCodeRegister int32  `gorm:"column:need_invitation_code_register;default:0" json:"needInvitationCodeRegister"`
-}
-
-type Applet struct {
-	ID         string    `gorm:"column:id;primary_key;size:64"`
-	Name       string    `gorm:"column:name;uniqueIndex;size:64"`
-	AppID      string    `gorm:"column:app_id;uniqueIndex;size:255"`
-	Icon       string    `gorm:"column:icon;size:255"`
-	URL        string    `gorm:"column:url;size:255"`
-	MD5        string    `gorm:"column:md5;size:255"`
-	Size       int64     `gorm:"column:size"`
-	Version    string    `gorm:"column:version;size:64"`
-	Priority   uint32    `gorm:"column:priority;size:64"`
-	Status     uint8     `gorm:"column:status"`
-	CreateTime time.Time `gorm:"column:create_time;autoCreateTime;size:64"`
-}

tools/data-conversion/chat/v2/chat.go

@@ -1,109 +0,0 @@
-// Copyright © 2023 OpenIM. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package v2
-
-import (
-	"time"
-)
-
-// Register Registration information sheet
-type Register struct {
-	UserID      string    `gorm:"column:user_id;primary_key;type:char(64)" json:"userID"`
-	DeviceID    string    `gorm:"column:device_id;type:varchar(255)"       json:"deviceID"`
-	IP          string    `gorm:"column:ip;type:varchar(32)"               json:"ip"`
-	Platform    string    `gorm:"column:platform;type:varchar(32)"         json:"platform"`
-	AccountType string    `gorm:"column:account_type;type:varchar(32)"     json:"accountType"` //email phone account
-	Mode        string    `gorm:"column:mode;type:varchar(32)"`                                //user admin
-	CreateTime  time.Time `gorm:"column:create_time"                       json:"createTime"`
-}
-
-// Account username and password table
-type Account struct {
-	UserID         string    `gorm:"column:user_id;primary_key;type:char(64)" json:"userID"`
-	Password       string    `gorm:"column:password;type:varchar(255)"        json:"password"`
-	CreateTime     time.Time `gorm:"column:create_time"                       json:"createTime"`
-	ChangeTime     time.Time `gorm:"column:change_time"                       json:"changeTime"`
-	OperatorUserID string    `gorm:"column:operator_user_id;type:varchar(64)" json:"operatorUserID"`
-}
-
-// Attribute user information table
-type Attribute struct {
-	UserID         string    `gorm:"column:user_id;primary_key;type:char(64)" json:"userID"`
-	Account        string    `gorm:"column:account;type:char(64)"             json:"account"`
-	PhoneNumber    string    `gorm:"column:phone_number;type:varchar(32)"     json:"phoneNumber"`
-	AreaCode       string    `gorm:"column:area_code;type:varchar(8)"         json:"areaCode"`
-	Email          string    `gorm:"column:email;type:varchar(64)"            json:"email"`
-	Nickname       string    `gorm:"column:nickname;type:varchar(64)"         json:"nickname"`
-	FaceURL        string    `gorm:"column:face_url;type:varchar(255)"        json:"faceURL"`
-	Gender         int32     `gorm:"column:gender"                            json:"gender"`
-	Birth          uint32    `gorm:"column:birth"                             json:"birth"`
-	CreateTime     time.Time `gorm:"column:create_time"                       json:"createTime"`
-	ChangeTime     time.Time `gorm:"column:change_time"                       json:"changeTime"`
-	BirthTime      time.Time `gorm:"column:birth_time"                        json:"birthTime"`
-	Level          int32     `gorm:"column:level;default:1"                   json:"level"`
-	AllowVibration int32     `gorm:"column:allow_vibration;default:1"         json:"allowVibration"`
-	AllowBeep      int32     `gorm:"column:allow_beep;default:1"              json:"allowBeep"`
-	AllowAddFriend int32     `gorm:"column:allow_add_friend;default:1"        json:"allowAddFriend"`
-}
-
-// User friend relationship table
-type ForbiddenAccount struct {
-	UserID         string    `gorm:"column:user_id;index:userID;primary_key;type:char(64)" json:"userID"`
-	CreateTime     time.Time `gorm:"column:create_time"                                    json:"createTime"`
-	Reason         string    `gorm:"column:reason;type:varchar(255)"                       json:"reason"`
-	OperatorUserID string    `gorm:"column:operator_user_id;type:varchar(255)"             json:"operatorUserID"`
-}
-
-// user login record table
-type UserLoginRecord struct {
-	UserID    string    `gorm:"column:user_id;size:64"             json:"userID"`
-	LoginTime time.Time `gorm:"column:login_time"                  json:"loginTime"`
-	IP        string    `gorm:"column:ip;type:varchar(32)"         json:"ip"`
-	DeviceID  string    `gorm:"column:device_id;type:varchar(255)" json:"deviceID"`
-	Platform  string    `gorm:"column:platform;type:varchar(32)"   json:"platform"`
-}
-
-// ip login registration is prohibited
-type IPForbidden struct {
-	IP            string    `gorm:"column:ip;primary_key;type:char(32)" json:"ip"`
-	LimitRegister int32     `gorm:"column:limit_register"               json:"limitRegister"`
-	LimitLogin    int32     `gorm:"column:limit_login"                  json:"limitLogin"`
-	CreateTime    time.Time `gorm:"column:create_time"                  json:"createTime"`
-}
-
-// Restrict userids to certain ip addresses
-type LimitUserLoginIP struct {
-	UserID     string    `gorm:"column:user_id;primary_key;type:char(64)" json:"userID"`
-	IP         string    `gorm:"column:ip;primary_key;type:char(32)"      json:"ip"`
-	CreateTime time.Time `gorm:"column:create_time"                       json:"createTime"`
-}
-
-// The invitation code is registered for use
-type InvitationRegister struct {
-	InvitationCode string    `gorm:"column:invitation_code;primary_key;type:char(32)" json:"invitationCode"`
-	CreateTime     time.Time `gorm:"column:create_time"                               json:"createTime"`
-	UsedByUserID   string    `gorm:"column:user_id;index:userID;type:char(64)"        json:"usedByUserID"`
-}
-
-type SignalRecord struct {
-	FileName    string    `gorm:"column:file_name;primary_key;type:char(128)"            json:"fileName"`
-	MediaType   string    `gorm:"column:media_type;type:char(64);index:media_type_index" json:"mediaType"`
-	RoomType    string    `gorm:"column:room_type;type:char(20)"                         json:"roomType"`
-	SenderID    string    `gorm:"column:sender_id;type:char(64);index:sender_id_index"   json:"senderID"`
-	RecvID      string    `gorm:"column:recv_id;type:char(64);index:recv_id_index"       json:"recvID"`
-	GroupID     string    `gorm:"column:group_id;type:char(64)"                          json:"groupID"`
-	DownloadURL string    `gorm:"column:download_url;type:text"                          json:"downloadURL"`
-	CreateTime  time.Time `gorm:"create_time;index:create_time_index"                    json:"createTime"`
-}

tools/data-conversion/chat/v3/admin/admin.go

@@ -1,34 +0,0 @@
-// Copyright © 2023 OpenIM open source community. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package admin
-
-import (
-	"time"
-)
-
-// Admin Background administrator.
-type Admin struct {
-	Account    string    `gorm:"column:account;primary_key;type:varchar(64)"`
-	Password   string    `gorm:"column:password;type:varchar(64)"`
-	FaceURL    string    `gorm:"column:face_url;type:varchar(255)"`
-	Nickname   string    `gorm:"column:nickname;type:varchar(64)"`
-	UserID     string    `gorm:"column:user_id;type:varchar(64)"` // openIM userID
-	Level      int32     `gorm:"column:level;default:1"  `
-	CreateTime time.Time `gorm:"column:create_time"`
-}
-
-func (Admin) TableName() string {
-	return "admins"
-}