From 409071344ca4b70aac99f655b85da070c830de88 Mon Sep 17 00:00:00 2001 From: "Xinwei Xiong (cubxxw)" <3293172751nss@gmail.com> Date: Tue, 2 Apr 2024 16:01:45 +0800 Subject: [PATCH] feat: Enhancements to OpenIM Engineering Practices with Standardizer and Tool Versioning (#2159) * feat: add standardizer optimize makefile * feat: add standardizer optimize makefile * feat: add openim test docs * feat: add openim kafka docs * feat: add openim kafka docs * feat: add openim kafka docs --- .github/code-language-detector.yml | 1 + .github/release-drafter.yml | 2 +- .github/standardizer.yml | 50 ++++++ .github/workflows/openimci.yml | 3 + Makefile | 1 + README_zh_CN.md | 2 +- docker-compose-1.yml | 2 +- docker-compose.yml | 2 +- docs/contrib/kafka.md | 162 ++++++++++++++++++ docs/contrib/logging.md | 2 +- docs/contrib/test.md | 86 +++++++++- docs/images/Open-IM.png | Bin 17614 -> 0 bytes .../{Architecture.jpg => architecture.jpg} | Bin docs/images/{Wechat.jpg => wechat.jpg} | Bin .../{directResolver.go => direct_resolver.go} | 0 scripts/make-rules/common-versions.mk | 58 +++++++ scripts/make-rules/common.mk | 7 + scripts/make-rules/tools.mk | 110 +++++++----- scripts/verify-standardizer.sh | 33 ++++ .../{sendMessage.json => send-message.json} | 0 20 files changed, 471 insertions(+), 50 deletions(-) create mode 100644 .github/standardizer.yml create mode 100644 docs/contrib/kafka.md delete mode 100644 docs/images/Open-IM.png rename docs/images/{Architecture.jpg => architecture.jpg} (100%) rename docs/images/{Wechat.jpg => wechat.jpg} (100%) rename pkg/common/discoveryregister/direct/{directResolver.go => direct_resolver.go} (100%) create mode 100644 scripts/make-rules/common-versions.mk create mode 100755 scripts/verify-standardizer.sh rename test/testdata/requests/{sendMessage.json => send-message.json} (100%) diff --git a/.github/code-language-detector.yml b/.github/code-language-detector.yml index 32a8c1f54..194c2474a 100644 --- a/.github/code-language-detector.yml +++ b/.github/code-language-detector.yml @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# https://github.com/marketplace/actions/code-language-detector directory: ./ file_types: - .go diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index 154386ff5..55ee241d7 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -48,4 +48,4 @@ template: | ## Contributors to this $REPOSITORY release - $CONTRIBUTORS \ No newline at end of file + $CONTRIBUTORS diff --git a/.github/standardizer.yml b/.github/standardizer.yml new file mode 100644 index 000000000..fceb69df1 --- /dev/null +++ b/.github/standardizer.yml @@ -0,0 +1,50 @@ +# https://github.com/marketplace/actions/conformity-checker-for-project +baseConfig: + searchDirectory: "./" + ignoreCase: false + +directoryNaming: + allowHyphens: true + allowUnderscores: false + mustBeLowercase: true + +fileNaming: + allowHyphens: true + allowUnderscores: true + mustBeLowercase: true + +ignoreFormats: + - "\\.log$" + - "\\.env$" + - "README\\.md$" + - "_test\\.go$" + - "\\.md$" + - _test\\.txt$ + - LICENSE + - Dockerfile + - CODEOWNERS + - Makefile + +ignoreDirectories: + - "vendor" + - ".git" + - "deployments" + - "node_modules" + - "logs" + - "CHANGELOG" + - "components" + - "_output" + - "tools/openim-web" + - "CHANGELOG" + - "examples/Test_directory" + - test/testdata + +fileTypeSpecificNaming: + ".yaml": + allowHyphens: true + allowUnderscores: false + mustBeLowercase: true + ".go": + allowHyphens: false + allowUnderscores: true + mustBeLowercase: true \ No newline at end of file diff --git a/.github/workflows/openimci.yml b/.github/workflows/openimci.yml index 5aeddd09a..f47283997 100644 --- a/.github/workflows/openimci.yml +++ b/.github/workflows/openimci.yml @@ -73,6 +73,9 @@ jobs: - name: Code Typecheck Detector uses: kubecub/typecheck@main + - name: Conformity Checker for Project + uses: kubecub/standardizer@main + - name: Module Operations run: | sudo make tidy diff --git a/Makefile b/Makefile index 89b9e4152..818372fb8 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ VERSION_PACKAGE=github.com/openimsdk/open-im-server/v3/pkg/version # Includes include scripts/make-rules/common.mk # make sure include common.mk at the first include line +include scripts/make-rules/common-versions.mk include scripts/make-rules/golang.mk include scripts/make-rules/image.mk include scripts/make-rules/copyright.mk diff --git a/README_zh_CN.md b/README_zh_CN.md index 7eabfa509..f42031165 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -52,7 +52,7 @@
## 🟢 扫描微信进群交流 -
+
## Ⓜ️ 关于 OpenIM
diff --git a/docker-compose-1.yml b/docker-compose-1.yml
index 39fc944ce..ed852fd29 100644
--- a/docker-compose-1.yml
+++ b/docker-compose-1.yml
@@ -1,4 +1,4 @@
-#fixme Clone openIM Server project before using docker-compose,project address:https://github.com/OpenIMSDK/Open-IM-Server.git
+#fixme Clone openIM Server project before using docker-compose,project address:https://github.com/openimsdk/open-im-server.git
# The command that triggers this file to pull the image is "docker compose up -f"
version: '3'
diff --git a/docker-compose.yml b/docker-compose.yml
index ef0714329..a983acbaf 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,4 +1,4 @@
-#fixme Clone openIM Server project before using docker-compose,project address:https://github.com/OpenIMSDK/Open-IM-Server.git
+#fixme Clone openIM Server project before using docker-compose,project address:https://github.com/openimsdk/open-im-server.git
# The command that triggers this file to pull the image is "docker compose up -d".
version: '3'
diff --git a/docs/contrib/kafka.md b/docs/contrib/kafka.md
new file mode 100644
index 000000000..4547c9480
--- /dev/null
+++ b/docs/contrib/kafka.md
@@ -0,0 +1,162 @@
+# OpenIM Kafka Guide
+
+This document aims to provide a set of concise guidelines to help you quickly install and use Kafka through Docker Compose.
+
+## Installing Kafka
+
+With the Docker Compose script provided by OpenIM, you can easily install Kafka. Use the following command to start Kafka:
+
+```bash
+docker compose up -d
+```
+
+After executing this command, Kafka will be installed and started. You can confirm the Kafka container is running with the following command:
+
+```bash
+docker ps | grep kafka
+```
+
+The output of this command, as shown below, displays the status information of the Kafka container:
+
+```
+be416b5a0851 bitnami/kafka:3.5.1 "/opt/bitnami/script…" 3 days ago Up 2 days 9092/tcp, 0.0.0.0:19094->9094/tcp, :::19094->9094/tcp kafka
+```
+
+### References
+
+- Official Docker installation documentation: [Click here](http://events.jianshu.io/p/b60afa35303a)
+- Detailed installation guide: [Tutorial on Towards Data Science](https://towardsdatascience.com/how-to-install-apache-kafka-using-docker-the-easy-way-4ceb00817d8b)
+
+## Using Kafka
+
+### Entering the Kafka Container
+
+To execute Kafka commands, you first need to enter the Kafka container. Use the following command:
+
+```bash
+docker exec -it kafka bash
+```
+
+### Kafka Command Tools
+
+Inside the Kafka container, you can use various command-line tools to manage Kafka. These tools include but are not limited to:
+
+- `kafka-topics.sh`: For creating, deleting, listing, or altering topics.
+- `kafka-console-producer.sh`: Allows sending messages to a specified topic from the command line.
+- `kafka-console-consumer.sh`: Allows reading messages from the command line, with the ability to specify topics.
+- `kafka-consumer-groups.sh`: For managing consumer group information.
+
+### Kafka Client Tool Installation
+
+For easier Kafka management, you can install Kafka client tools. If you installed Kafka through OpenIM's Docker Compose, you can install the Kafka client tools with the following command:
+
+```bash
+make install.kafkactl
+```
+
+### Automatic Topic Creation
+
+When installing Kafka through OpenIM's Docker Compose method, OpenIM automatically creates the following topics:
+
+- `latestMsgToRedis`
+- `msgToPush`
+- `offlineMsgToMongoMysql`
+
+These topics are created using the `scripts/create-topic.sh` script. The script waits for Kafka to be ready before executing the commands to create topics:
+
+```bash
+# Wait for Kafka to be ready
+until /opt/bitnami/kafka/bin/kafka-topics.sh --list --bootstrap-server localhost:9092; do
+ echo "Waiting for Kafka to be ready..."
+ sleep 2
+done
+
+# Create topics
+/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 8 --topic latestMsgToRedis
+/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 8 --topic msgToPush
+/opt/bitnami/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 8 --topic offlineMsgToMongoMysql
+
+echo "Topics created."
+```
+
+The optimized and expanded documentation further details some basic commands for operations inside the Kafka container, as well as basic commands for managing Kafka using `kafkactl`. Here is a more detailed guide.
+
+
+## Basic Commands in the Kafka Container
+
+### Listing Topics
+
+To list all existing topics, you can use the following command:
+
+```bash
+kafka-topics.sh --list --bootstrap-server localhost:9092
+```
+
+### Creating a New Topic
+
+When creating a new topic, you can specify the number of partitions and the replication factor. Here is the command to create a new topic:
+
+```bash
+kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic your_topic_name
+```
+
+### Producing Messages
+
+To send messages to a specific topic, you can use the producer command. The following command prompts you to enter messages, which are sent to the specified topic with each press of the Enter key:
+
+```bash
+kafka-console-producer.sh --broker-list localhost:9092 --topic your_topic_name
+```
+
+### Consuming Messages
+
+To read messages from a specific topic, you can use the consumer command. The following command reads new messages from the specified topic and outputs them on the console:
+
+```bash
+kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic your_topic_name --from-beginning
+```
+
+The `
+
+--from-beginning` parameter reads messages from the beginning of the topic. If this parameter is omitted, only new messages will be read.
+
+
+## Basic Commands Using `kafkactl`
+
+`kafkactl` is a command-line tool for managing and operating Kafka clusters. It offers a more modern way to interact with Kafka.
+
+### Listing Topics
+
+To list all topics, you can use:
+
+```bash
+kafkactl get topics
+```
+
+### Creating a New Topic
+
+To create a new topic with `kafkactl`, use:
+
+```bash
+kafkactl create topic your_topic_name --partitions 1 --replication-factor 1
+```
+
+### Producing Messages
+
+To send messages to a topic, you can use:
+
+```bash
+kafkactl produce your_topic_name --value "your message"
+```
+
+Here, `"your message"` is the content of the message you want to send.
+
+### Consuming Messages
+
+To consume messages from a topic, use:
+
+```bash
+kafkactl consume your_topic_name --from-beginning
+```
+
+Again, the `--from-beginning` parameter will start consuming messages from the beginning of the topic. If you do not wish to start from the beginning, you can omit this parameter.
\ No newline at end of file
diff --git a/docs/contrib/logging.md b/docs/contrib/logging.md
index e25ac92c2..b88fbeb8d 100644
--- a/docs/contrib/logging.md
+++ b/docs/contrib/logging.md
@@ -2,7 +2,7 @@
## Script Logging Documentation Link
-If you wish to view the script's logging documentation, you can click on this link: [Logging Documentation](https://github.com/OpenIMSDK/Open-IM-Server/blob/main/docs/contrib/bash-log.md).
+If you wish to view the script's logging documentation, you can click on this link: [Logging Documentation](https://github.com/openimsdk/open-im-server/blob/main/docs/contrib/bash-log.md).
Below is the documentation for logging and error handling in the OpenIM Go project.
diff --git a/docs/contrib/test.md b/docs/contrib/test.md
index 37c1792cf..2470a0537 100644
--- a/docs/contrib/test.md
+++ b/docs/contrib/test.md
@@ -2,14 +2,94 @@
This document serves as a comprehensive guide to understanding and utilizing the `test.sh` script for testing OpenIM RPC services. The `test.sh` script is a collection of bash functions designed to test various aspects of the OpenIM RPC services, ensuring that each part of the API is functioning as expected.
-+ Scripts:https://github.com/OpenIMSDK/Open-IM-Server/tree/main/scripts/install/test.sh
++ Scripts:https://github.com/openimsdk/open-im-server/tree/main/scripts/install/test.sh
-For some complex, bulky functional tests, performance tests, and various e2e tests, We are all in the current warehouse to https://github.com/OpenIMSDK/Open-IM-Server/tree/main/test or https://github.com/openim-sigs/test-infra directory In the.
+For some complex, bulky functional tests, performance tests, and various e2e tests, We are all in the current warehouse to https://github.com/openimsdk/open-im-server/tree/main/test or https://github.com/openim-sigs/test-infra directory In the.
+ About OpenIM Feature [Test Docs](https://docs.google.com/spreadsheets/d/1zELWkwxgOOZ7u5pmYCqqaFnvZy2SVajv/edit?usp=sharing&ouid=103266350914914783293&rtpof=true&sd=true)
+## Util Test
-## Usage
+Let's restructure and enhance the document under a unified second-level heading, adding clarity and details for better comprehension and visual appeal.
+
+---
+
+## Development Guide
+
+### Comprehensive Testing Instructions
+
+#### Running Unit Tests
+
+- **Command**: To execute unit tests, input the following in your terminal:
+ ```
+ make test
+ ```
+
+#### Evaluating Test Coverage
+
+- **Overview**: It's crucial to assess how much of your code is covered by tests.
+- **Command**:
+ ```bash
+ make cover
+ ```
+ This command generates a report detailing the percentage of your code tested, ensuring adherence to quality standards.
+
+#### Conducting API Tests
+
+- **Purpose**: API tests validate the interaction and functionality of your application's interfaces.
+- **How to Run**:
+ ```
+ make test-api
+ ```
+ Use this to check the integrity and reliability of your API endpoints.
+
+#### End-to-End (E2E) Testing
+
+- **Scope**: E2E tests simulate real-user scenarios from start to finish.
+- **Execution**:
+ ```
+ make test-e2e
+ ```
+ This comprehensive testing ensures your application performs as expected in real-world situations.
+
+### Crafting Unit Test Cases
+
+#### Setup for Test Case Generation
+
+- **Installation**: Install the `gotests` tool to generate test cases automatically.
+ ```bash
+ make install.gotests
+ ```
+ This command installs the `gotests` tool for test case generation.
+
+- **Environment Preparation**: Define your test template environment variable and generate test cases as shown below:
+ ```bash
+ export GOTESTS_TEMPLATE=testify
+ gotests -i -w -only keyFunc .
+ ```
+ This prepares your environment for test case generation using the `testify` template.
+
+#### Isolating Function Tests
+
+- **Single Function Testing**: When you need to focus on testing a single function for detailed examination.
+- **Method**:
+ ```bash
+ go test -v -run TestKeyFunc
+ ```
+ This command specifically runs tests for `TestKeyFunc`, allowing targeted debugging and validation.
+
+### Important Note
+
+- **Quality Assurance**: Throughout your development process, it is imperative to ensure that the unit test coverage meets or surpasses the standards set by OpenIM.
+- **Maintaining Standards**: Regularly running your tests with
+ ```make test```
+ supports maintaining high code quality and adherence to OpenIM's rigorous testing benchmarks.
+
+## E2E Test
+
+TODO
+
+## Api Test
The `test.sh` script is located within the `./scripts/install/` directory of the OpenIM service's codebase. To use the script, navigate to this directory from your terminal:
diff --git a/docs/images/Open-IM.png b/docs/images/Open-IM.png
deleted file mode 100644
index b617065505626ea5df9467ca5e945f851258aee6..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 17614
zcmZU*2RNJU8#Z3r(we3AtWs)Io1nF+)vAas6t!zhtkRa+_1d+HqLmpsu(y6z`JUss)+go)(FjT_{e8Y(Yu+#rPG|Nd}?
z2>)rA3sJ)VBJg~vu5_bhfMo;!<(B<3oo6?0l*N(az_;<=@4nSA_PlX}qV3NQL8sgI
z*Eepc*l4Oed*x@b6L^sT?1Zdc5yT(*<2(=B5HMGYis7g@9X2O
zW?J;q3GHS%ol(Vj00ci0`PU=8_vTVqE|w^R6TMktWYvJpUjNrfFtUvcMOr>;ku44s
zg_kxQnt0gVaN!ctd<{%W>D|CMIzAE{@!1P%>&d= mxNU+6(j_{8x`ltP|#9Hx=xpc4(RFtqZ#y?!<@6wPkO9NYC82=fbwQU&9gH@jxrfP%O-ltnSNZJ7Tld1bu`(zdT*Z
zR{SwiFd$dcz)*VLr-k0{;V;Fbcm7{5(aII<-{|x|vsGrc;AptxpVZkCQ#LuUBK!6{
zs9W%qbD^xt>6-hCf;yt3$F1{nldE(O+%K%i(q@urDNJ%tOmvcO-k6S
zU{}-yMH()vO5aeA1V3ugvaZ?@g?~u^K{5R+nr?q(75$i630&pGOya4eQ9f3!v%vFg
zaBnWt(4zWA)GCifm9?Q`r6PGz5tF8_b@;NmGS*x-JK1g4FT0UL#Z@{m*-BzK9XsRR
zEZfavLIzvw94&{HM3hqWUX;g{;_dg}?f2~WN&yI!eu(%Y3jb)eqk_NEci~dRAD!u0
zG;3XHHkDP#O0TV=!ySzs~q?
zlaA3Xh?qG}R4NPbICOICioC9`cs}@!+c%$|(so4(r{$Ri4OHBOo)=$q$!!ltnp?=a
z;>EtXfp_(*k@h4zHU(R4KmARVbNN%}1Jl-6XEya^@ApHWBi}s8oVv?B6Sa1*1_43p
z4GwHuEyASVUMoN3(2td&^k`7rFBMGPr8`9YHPULX@sxo+-~g*7Lv+hY#C);#6VlW)
z7t|G&?K_2hX|7p%_T9)V!ET}^$!xG-ZRUiCJE3dhq8K}Nzgdr!683!p$HH&t(%sev
zP0x(A@u@G|lQVzr>g*^axq^4d$K&F@D)C2-V_kc%9!GA8%=NmS>nnqIYt8^Re+7;4
zf-YEdv?=G-7kh_R7DjlC2A}Y>ji>YcBE+H=HhNHnah`SbhS+{gF3Pj_vZPZx(o(9q
zyO+S(HV?e(+a{pVFm+)FjWiekXnTbB6IvaJFBgHkkW1{$5=Y(vV#K!s{ri$P#VK({
zsjvdA7INKdJu%WNM{GgiP4sYAHYiBc0;fA*&)@j5mUUeZ6WB5AA6oSZOkOBeTMsBZ
zyNpUL@C&j;6pZ-y-ryHYnW!4BY30m?;i0*&k>nH0aixu7&EF7a+a?v7jecE`DGPC=
zGTXS{`P=$eyHdkEvhuYr qrj{4+19;`O(UX6w|hhIN#>W`(7$r*W7iz=OYv
zFNsb8aKU7ws4Zbl!^`Y?Gkz%&@=pbY%{N0Ol7nIsl7rNlR-I%6dyXnV$cZqOMp4V2
z-a2X-_A(jo_SBazD@Ab%Z|tMy`*7pil4h1P7jw`CKF;QI)*Vbj!>0|nBYQanmsUAq
zKkjxex4&(2F$NSM0aHwhoz!$ZFLQK#3D
;L41XyR}%dSC+umejfPfpc=)y=F5gxYxkZ)tT?qf;(a~~
zn58}ekHwhw2)xM_yfXB?NO)4{pZNwdhrNQML^rc+ao2<$2+`d99JI$h=hR`Z*&j9p
z{aGcp=56pP#7bDpIRRB+s;_`sxTBcm!oa6;K#u1)mS4PQb