diff --git a/.github/workflows/docker-build-and-release-services-images.yml b/.github/workflows/docker-build-and-release-services-images.yml index ed4c7ad38..407589f1e 100644 --- a/.github/workflows/docker-build-and-release-services-images.yml +++ b/.github/workflows/docker-build-and-release-services-images.yml @@ -19,26 +19,26 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3.8.0 - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Aliyun Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: registry.cn-hangzhou.aliyuncs.com username: ${{ secrets.ALIREGISTRY_USERNAME }} @@ -46,7 +46,7 @@ jobs: - name: Extract metadata for Docker (tags, labels) id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v5.6.0 with: tags: | type=ref,event=tag @@ -54,7 +54,6 @@ jobs: type=ref,event=branch type=semver,pattern={{version}} type=semver,pattern=v{{version}} - # type=semver,pattern={{major}}.{{minor}} type=semver,pattern=release-{{raw}} type=sha type=raw,value=${{ github.event.inputs.tag }} diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 10a4154d6..4033603e6 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -4,7 +4,7 @@ on: push: pull_request: paths-ignore: - - '**/*.md' + - "**/*.md" workflow_dispatch: @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - go_version: ["1.21.x", "1.22.x"] + go_version: ["1.22.x"] steps: - name: Checkout Server repository @@ -37,27 +37,20 @@ jobs: - name: Set up infra services uses: hoverkraft-tech/compose-action@v2.0.1 - # Uncomment and set the correct path to your docker-compose file with: compose-file: "./docker-compose.yml" - # run: | - # sudo docker compose up -d - # sudo sleep 30 # Increased sleep time for better stability - # timeout-minutes: 60 # Increased timeout for Docker setup + # - name: Get Internal IP Address + # id: get-ip + # run: | + # IP=$(hostname -I | awk '{print $1}') + # echo "The IP Address is: $IP" + # echo "::set-output name=ip::$IP" - - # - name: Get Internal IP Address - # id: get-ip - # run: | - # IP=$(hostname -I | awk '{print $1}') - # echo "The IP Address is: $IP" - # echo "::set-output name=ip::$IP" - - # - name: Update .env - # run: | - # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml - # cat config/minio.yml + # - name: Update .env + # run: | + # sed -i 's|externalAddress:.*|externalAddress: "http://${{ steps.get-ip.outputs.ip }}:10005"|' config/minio.yml + # cat config/minio.yml - name: Build and test Server Services run: | @@ -85,6 +78,90 @@ jobs: mage start mage check + - name: Test Server and Chat + run: | + check_error() { + echo "Response: $1" + errCode=$(echo $1 | jq -r '.errCode') + if [ "$errCode" != "0" ]; then + errMsg=$(echo $1 | jq -r '.errMsg') + echo "Error: $errMsg" + exit 1 + fi + } + + # Test register + response1=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "verifyCode": "666666", + "platform": 3, + "autoLogin": true, + "user":{ + "nickname": "test12312", + "areaCode":"+86", + "phoneNumber": "12345678190", + "password":"test123456" + } + }' http://127.0.0.1:10008/account/register) + check_error "$response1" + userID1=$(echo $response1 | jq -r '.data.userID') + echo "userID1: $userID1" + + response2=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "verifyCode": "666666", + "platform": 3, + "autoLogin": true, + "user":{ + "nickname": "test22312", + "areaCode":"+86", + "phoneNumber": "12345678290", + "password":"test123456" + } + }' http://127.0.0.1:10008/account/register) + check_error "$response2" + userID2=$(echo $response2 | jq -r '.data.userID') + echo "userID2: $userID2" + + # Test login + login_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "platform": 3, + "areaCode":"+86", + "phoneNumber": "12345678190", + "password":"test123456" + }' http://localhost:10008/account/login) + check_error "$login_response" + + # Test get admin token + get_admin_token_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -d '{ + "secret": "openIM123", + "platformID": 2, + "userID": "imAdmin" + }' http://127.0.0.1:10002/auth/get_admin_token) + check_error "$get_admin_token_response" + adminToken=$(echo $get_admin_token_response | jq -r '.data.token') + echo "adminToken: $adminToken" + + # Test send message + send_msg_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -H "token: $adminToken" -d '{ + "sendID": "'$userID1'", + "recvID": "'$userID2'", + "senderPlatformID": 3, + "content": { + "content": "hello!!" + }, + "contentType": 101, + "sessionType": 1 + }' http://127.0.0.1:10002/msg/send_msg) + check_error "$send_msg_response" + + # Test get users + get_users_response=$(curl -X POST -H "Content-Type: application/json" -H "operationID: imAdmin" -H "token: $adminToken" -d '{ + "pagination": { + "pageNumber": 1, + "showNumber": 100 + } + }' http://127.0.0.1:10002/user/get_users) + check_error "$get_users_response" + go-test: name: Benchmark Test with go ${{ matrix.go_version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -93,11 +170,11 @@ jobs: env: SDK_DIR: openim-sdk-core CONFIG_PATH: config/notification.yml - # pull-requests: write + strategy: matrix: - os: [ ubuntu-latest ] - go_version: [ "1.22.x" ] + os: [ubuntu-latest] + go_version: ["1.22.x"] steps: - name: Checkout Server repository @@ -106,7 +183,8 @@ jobs: - name: Checkout SDK repository uses: actions/checkout@v4 with: - repository: 'openimsdk/openim-sdk-core' + repository: "openimsdk/openim-sdk-core" + ref: "release-v3.8" path: ${{ env.SDK_DIR }} - name: Set up Go ${{ matrix.go_version }} @@ -119,11 +197,6 @@ jobs: go install github.com/magefile/mage@latest go mod download - - name: Install yq - run: | - sudo wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - name: Modify Server Configuration run: | yq e '.groupCreated.isSendMsg = true' -i ${{ env.CONFIG_PATH }} @@ -183,11 +256,3 @@ jobs: run: | CONTAINER_NAME="${{ github.event.repository.name }}-container" docker logs $CONTAINER_NAME - - # - name: Cleanup Docker Container - # run: | - # CONTAINER_NAME="${{ github.event.repository.name }}-container" - # IMAGE_NAME="${{ github.event.repository.name }}-test" - # docker stop $CONTAINER_NAME - # docker rm $CONTAINER_NAME - # docker rmi $IMAGE_NAME diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml index 44b4f81f4..1f5762ccb 100644 --- a/.github/workflows/merge-from-milestone.yml +++ b/.github/workflows/merge-from-milestone.yml @@ -1,4 +1,4 @@ -name: Create Pre-Release PR from Milestone +name: Create Individual PRs from Milestone permissions: contents: write @@ -9,24 +9,24 @@ on: workflow_dispatch: inputs: milestone_name: - description: 'Milestone name to collect closed PRs from' + description: "Milestone name to collect closed PRs from" required: true - default: 'v3.8.2' + default: "v3.8.4" target_branch: - description: 'Target branch to merge the consolidated PR' + description: "Target branch to merge the consolidated PR" required: true - default: 'pre-release-v3.8.2' + default: "pre-release-v3.8.4" env: - MILESTONE_NAME: ${{ github.event.inputs.milestone_name || 'v3.8.2' }} - TARGET_BRANCH: ${{ github.event.inputs.target_branch || 'pre-release-v3.8.2' }} + MILESTONE_NAME: ${{ github.event.inputs.milestone_name || 'v3.8.4' }} + TARGET_BRANCH: ${{ github.event.inputs.target_branch || 'pre-release-v3.8.4' }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BOT_TOKEN: ${{ secrets.BOT_TOKEN }} LABEL_NAME: cherry-picked - TEMP_DIR: /tmp # Using /tmp as the temporary directory + TEMP_DIR: /tmp jobs: - cherry_pick_milestone_prs: + merge_milestone_prs: runs-on: ubuntu-latest steps: - name: Setup temp directory @@ -47,7 +47,6 @@ jobs: - name: Setup Git User for OpenIM-Robot run: | - # Set up Git credentials for the bot git config --global user.email "OpenIM-Robot@users.noreply.github.com" git config --global user.name "OpenIM-Robot" @@ -83,136 +82,84 @@ jobs: if ! echo "$labels" | grep -q "${LABEL_NAME}"; then echo "PR #$pr_number does not have the 'cherry-picked' label. Adding to the list." echo "$pr_number" >> ${{ env.TEMP_DIR }}/pr_numbers.txt - else - echo "PR #$pr_number already has the 'cherry-picked' label. Skipping." fi done - # Sort the filtered PR numbers sort -n ${{ env.TEMP_DIR }}/pr_numbers.txt -o ${{ env.TEMP_DIR }}/pr_numbers.txt - echo "Filtered and sorted PR numbers:" - cat ${{ env.TEMP_DIR }}/pr_numbers.txt || echo "No closed PR numbers found for milestone." - - - name: Fetch Merge Commits for PRs and Generate Title and Body + - name: Create Individual PRs run: | - # Ensure the files are initialized - > ${{ env.TEMP_DIR }}/commit_hashes.txt - > ${{ env.TEMP_DIR }}/pr_title.txt - > ${{ env.TEMP_DIR }}/pr_body.txt - - # Write description to the PR body - echo "### Description:" >> ${{ env.TEMP_DIR }}/pr_body.txt - echo "Merging PRs from milestone \`$MILESTONE_NAME\` into target branch \`$TARGET_BRANCH\`." >> ${{ env.TEMP_DIR }}/pr_body.txt - echo "" >> ${{ env.TEMP_DIR }}/pr_body.txt - echo "### Need Merge PRs:" >> ${{ env.TEMP_DIR }}/pr_body.txt - - pr_numbers_in_title="" - - # Process sorted PR numbers and generate commit hashes for pr_number in $(cat ${{ env.TEMP_DIR }}/pr_numbers.txt); do - echo "Processing PR #$pr_number" pr_details=$(curl -s -H "Authorization: token $BOT_TOKEN" \ -H "Accept: application/vnd.github+json" \ "https://api.github.com/repos/${{ github.repository }}/pulls/$pr_number") pr_title=$(echo "$pr_details" | jq -r '.title') + pr_body=$(echo "$pr_details" | jq -r '.body') + pr_creator=$(echo "$pr_details" | jq -r '.user.login') merge_commit=$(echo "$pr_details" | jq -r '.merge_commit_sha') short_commit_hash=$(echo "$merge_commit" | cut -c 1-7) - # Append PR details to the body - echo "- $pr_title: (#$pr_number) ($short_commit_hash)" >> ${{ env.TEMP_DIR }}/pr_body.txt + if [ "$merge_commit" != "null" ]; then + git fetch origin + + echo "Checking out target branch: $TARGET_BRANCH" + git checkout $TARGET_BRANCH - if [ "$merge_commit" != "null" ];then - echo "$merge_commit" >> ${{ env.TEMP_DIR }}/commit_hashes.txt - echo "#$pr_number" >> ${{ env.TEMP_DIR }}/pr_title.txt - pr_numbers_in_title="$pr_numbers_in_title #$pr_number" + echo "Pulling latest changes from target branch: $TARGET_BRANCH" + git pull origin $TARGET_BRANCH + + cherry_pick_branch="cherry-pick-${short_commit_hash}" + git checkout -b $cherry_pick_branch + + echo "Cherry-picking commit: $merge_commit" + if ! git cherry-pick "$merge_commit" --strategy=recursive -X theirs; then + echo "Conflict detected for $merge_commit. Resolving with incoming changes." + conflict_files=$(git diff --name-only --diff-filter=U) + echo "Conflicting files:" + echo "$conflict_files" + + for file in $conflict_files; do + if [ -f "$file" ]; then + echo "Resolving conflict for $file" + git add "$file" + else + echo "File $file has been deleted. Skipping." + git rm "$file" + fi + done + + echo "Conflicts resolved. Continuing cherry-pick." + git cherry-pick --continue || { echo "Cherry-pick failed, but continuing to create PR."; } + else + echo "Cherry-pick successful for commit $merge_commit." + fi + + git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" + + echo "Pushing branch: $cherry_pick_branch" + if ! git push origin $cherry_pick_branch --force; then + echo "Push failed, but continuing to create PR..." + fi + + new_pr_title="$pr_title [Created by @$pr_creator from #$pr_number]" + new_pr_body="$pr_body + > This PR is created from original PR #$pr_number." + + response=$(curl -s -X POST -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/${{ github.repository }}/pulls \ + -d "$(jq -n --arg title "$new_pr_title" \ + --arg head "$cherry_pick_branch" \ + --arg base "$TARGET_BRANCH" \ + --arg body "$new_pr_body" \ + '{title: $title, head: $head, base: $base, body: $body}')") + + new_pr_number=$(echo "$response" | jq -r '.number') + echo "Created PR #$new_pr_number" + + curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -d '{"labels": ["milestone-merge"]}' \ + "https://api.github.com/repos/${{ github.repository }}/issues/$new_pr_number/labels" fi done - - commit_hashes=$(cat ${{ env.TEMP_DIR }}/commit_hashes.txt | tr '\n' ' ') - first_commit_hash=$(head -n 1 ${{ env.TEMP_DIR }}/commit_hashes.txt) - cherry_pick_branch="cherry-pick-${first_commit_hash:0:7}" - echo "COMMIT_HASHES=$commit_hashes" >> $GITHUB_ENV - echo "CHERRY_PICK_BRANCH=$cherry_pick_branch" >> $GITHUB_ENV - echo "pr_numbers_in_title=$pr_numbers_in_title" >> $GITHUB_ENV - - - name: Pull and Cherry-pick Commits, Then Push - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BOT_TOKEN: ${{ secrets.BOT_TOKEN }} - run: | - # Fetch and pull the latest changes from the target branch - git fetch origin - git checkout $TARGET_BRANCH - git pull origin $TARGET_BRANCH - - # Create a new branch for cherry-picking - git checkout -b $CHERRY_PICK_BRANCH - - # Cherry-pick the commits and handle conflicts - for commit_hash in $COMMIT_HASHES; do - echo "Attempting to cherry-pick commit $commit_hash" - if ! git cherry-pick "$commit_hash" --strategy=recursive -X theirs; then - echo "Conflict detected for $commit_hash. Resolving with incoming changes." - conflict_files=$(git diff --name-only --diff-filter=U) - echo "Conflicting files:" - echo "$conflict_files" - - for file in $conflict_files; do - if [ -f "$file" ]; then - echo "Resolving conflict for $file" - git add "$file" - else - echo "File $file has been deleted. Skipping." - git rm "$file" - fi - done - - echo "Conflicts resolved. Continuing cherry-pick." - git cherry-pick --continue - else - echo "Cherry-pick successful for commit $commit_hash." - fi - done - - # Push the cherry-pick branch to the repository - git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" - git push origin $CHERRY_PICK_BRANCH --force - - - name: Create Pull Request - run: | - # Prepare and create the PR - pr_title="deps: Merge ${{ env.pr_numbers_in_title }} PRs into $TARGET_BRANCH" - pr_body=$(cat ${{ env.TEMP_DIR }}/pr_body.txt) - - echo "Prepared PR title:" - echo "$pr_title" - echo "Prepared PR body:" - echo "$pr_body" - - # Create the PR using the GitHub API - response=$(curl -s -X POST -H "Authorization: token $BOT_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${{ github.repository }}/pulls \ - -d "$(jq -n --arg title "$pr_title" \ - --arg head "$CHERRY_PICK_BRANCH" \ - --arg base "$TARGET_BRANCH" \ - --arg body "$pr_body" \ - '{title: $title, head: $head, base: $base, body: $body}')") - - pr_number=$(echo "$response" | jq -r '.number') - echo "$pr_number" > ${{ env.TEMP_DIR }}/created_pr_number.txt - echo "Created PR #$pr_number" - - - name: Add Label to Created Pull Request - run: | - # Add 'milestone-merge' label to the created PR - pr_number=$(cat ${{ env.TEMP_DIR }}/created_pr_number.txt) - echo "Adding label to PR #$pr_number" - - curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - -d '{"labels": ["milestone-merge"]}' \ - "https://api.github.com/repos/${{ github.repository }}/issues/$pr_number/labels" - - echo "Added 'milestone-merge' label to PR #$pr_number." diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index 40b79e61a..998a11cf3 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -25,11 +25,11 @@ jobs: with: path: main-repo - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3.3.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v3.8.0 - name: Build Docker image id: build @@ -38,11 +38,8 @@ jobs: context: ./main-repo load: true tags: "openim/openim-server:local" - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Save Docker image to file - run: docker save -o image.tar openim/openim-server:local + cache-from: type=gha,scope=build + cache-to: type=gha,mode=max,scope=build - name: Checkout compose repository uses: actions/checkout@v4 @@ -66,43 +63,12 @@ jobs: run: | cd ${{ github.workspace }}/compose-repo docker compose up -d - sleep 60 - # - name: Check openim-server health - # run: | - # timeout=300 - # interval=30 - # elapsed=0 - # while [[ $elapsed -le $timeout ]]; do - # if ! docker exec openim-server mage check; then - # echo "openim-server is not ready, waiting..." - # sleep $interval - # elapsed=$(($elapsed + $interval)) - # else - # echo "Health check successful" - # exit 0 - # fi - # done - # echo "Health check failed after 5 minutes" - # exit 1 - - # - name: Check openim-chat health - # if: success() - # run: | - # if ! docker exec openim-chat mage check; then - # echo "openim-chat check failed" - # exit 1 - # else - # echo "Health check successful" - # exit 0 - # fi - - - name: Load Docker image from file - run: docker load -i image.tar + docker compose ps - name: Extract metadata for Docker (tags, labels) id: meta - uses: docker/metadata-action@v5.5.1 + uses: docker/metadata-action@v5.6.0 with: images: | openim/openim-server @@ -112,29 +78,27 @@ jobs: type=ref,event=tag type=schedule type=ref,event=branch - type=semver,pattern={{version}} + # type=semver,pattern={{version}} type=semver,pattern=v{{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} type=semver,pattern=release-{{raw}} type=sha type=raw,value=${{ github.event.inputs.tag }} - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Aliyun Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3.3.0 with: registry: registry.cn-hangzhou.aliyuncs.com username: ${{ secrets.ALIREGISTRY_USERNAME }} @@ -148,3 +112,28 @@ jobs: platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=build + cache-to: type=gha,mode=max,scope=build + + - name: Verify multi-platform support + run: | + images=("openim/openim-server" "ghcr.io/openimsdk/openim-server" "registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server") + for image in "${images[@]}"; do + for tag in $(echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n'); do + manifest=$(docker manifest inspect "$image:$tag" || echo "error") + if [[ "$manifest" == "error" ]]; then + echo "Manifest not found for $image:$tag" + exit 1 + fi + amd64_found=$(echo "$manifest" | jq '.manifests[] | select(.platform.architecture == "amd64")') + arm64_found=$(echo "$manifest" | jq '.manifests[] | select(.platform.architecture == "arm64")') + if [[ -z "$amd64_found" ]]; then + echo "Multi-platform support check failed for $image:$tag - missing amd64" + exit 1 + fi + if [[ -z "$arm64_found" ]]; then + echo "Multi-platform support check failed for $image:$tag - missing arm64" + exit 1 + fi + done + done diff --git a/LICENSE b/LICENSE index 261eeb9e9..4591ca426 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,35 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +# Open Source License - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +OpenIM is licensed under the Apache License 2.0, with the following additional conditions: - 1. Definitions. +1. OpenIM may be utilized commercially, including as a backend service for other applications or as an application development platform for enterprises. +A commercial license must be obtained from the producer if: + - Under no circumstances may you operate a multi-tenant or multi-business environment using the OpenIM source code, whether or not you have modified the repository code. In other words, a single instance of OpenIM may not simultaneously serve multiple enterprises, nor may it serve multiple lines of business within the same enterprise. + - If you intend to operate in such a multi-tenant or multi-business manner, you must obtain a commercial license from the producer in advance. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +2. As a contributor, you should agree that: - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. + a. The producer can adjust the open-source agreement to be more strict or more relaxed as deemed necessary. + b. Your contributed code may be used for commercial purposes, including but not limited to its cloud business operations. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +Apart from the specific conditions mentioned above, all other rights and restrictions follow the Apache License 2.0. Detailed information about the Apache License 2.0 can be found at http://www.apache.org/licenses/LICENSE-2.0. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +For any licensing-related questions or to obtain a commercial license, please contact contact@openim.io. - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +© 2024 OpenIMSDK - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +---------- - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. + http://www.apache.org/licenses/LICENSE-2.0 - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 6b1779b25..3b88935eb 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,8 @@ Thank you for contributing to building a powerful instant messaging solution! ## :closed_book: License -OpenIMSDK is available under the Apache License 2.0. See the [LICENSE file](https://github.com/openimsdk/open-im-server/blob/main/LICENSE) for more information. +For more details, please refer to [here](./LICENSE). + diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index 3e08596ec..ae14d02a1 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -61,8 +61,12 @@ func (mc *OnlineHistoryMongoConsumerHandler) HandleChatWs2Mongo(ctx context.Cont } else { prommetrics.MsgInsertMongoSuccessCounter.Inc() } - var seqs []int64 - for _, msg := range msgFromMQ.MsgData { - seqs = append(seqs, msg.Seq) - } + //var seqs []int64 + //for _, msg := range msgFromMQ.MsgData { + // seqs = append(seqs, msg.Seq) + //} + //if err := mc.msgTransferDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs); err != nil { + // log.ZError(ctx, "remove cache msg from redis err", err, "msg", + // msgFromMQ.MsgData, "conversationID", msgFromMQ.ConversationID) + //} } diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 2ee4d6e1c..cd7839234 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -773,7 +773,7 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { continue } - seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-conversation.MsgDestructTime) + seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-(conversation.MsgDestructTime*1000)) if err != nil { return nil, err } diff --git a/internal/rpc/msg/revoke.go b/internal/rpc/msg/revoke.go index 97de0f48a..c2fb5833f 100644 --- a/internal/rpc/msg/revoke.go +++ b/internal/rpc/msg/revoke.go @@ -17,9 +17,10 @@ package msg import ( "context" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/protocol/constant" @@ -79,8 +80,10 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. switch members[req.UserID].RoleLevel { case constant.GroupOwner: case constant.GroupAdmin: - if members[msgs[0].SendID].RoleLevel != constant.GroupOrdinaryUsers { - return nil, errs.ErrNoPermission.WrapMsg("no permission") + if sendMember, ok := members[msgs[0].SendID]; ok { + if sendMember.RoleLevel != constant.GroupOrdinaryUsers { + return nil, errs.ErrNoPermission.WrapMsg("no permission") + } } default: return nil, errs.ErrNoPermission.WrapMsg("no permission") diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index 7cf0f9e0a..ee2a06f53 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -62,14 +62,13 @@ func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []st claims, err := tokenverify.GetClaimFromToken(token, authverify.Secret(a.accessSecret)) if err != nil { continue + } + key := cachekey.GetTokenKey(claims.UserID, claims.PlatformID) + if v, ok := setMap[key]; ok { + v[token] = constant.KickedToken } else { - key := cachekey.GetTokenKey(claims.UserID, claims.PlatformID) - if v, ok := setMap[key]; ok { - v[token] = constant.KickedToken - } else { - setMap[key] = map[string]any{ - token: constant.KickedToken, - } + setMap[key] = map[string]any{ + token: constant.KickedToken, } } } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 82841c41c..53dd7f13d 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -306,7 +306,7 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ log.ZError(ctx, "json.Unmarshal", err) return } - if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.Content == "" { + if quoteMsg.QuoteMessage == nil { return } if quoteMsg.QuoteMessage.Content == "e30=" { @@ -719,13 +719,13 @@ func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error if index <= 0 { return errs.ErrInternalServer.WrapMsg("docID is invalid", "docID", docID) } - index, err := strconv.Atoi(docID[index+1:]) + docIndex, err := strconv.Atoi(docID[index+1:]) if err != nil { return errs.WrapMsg(err, "strconv.Atoi", "docID", docID) } conversationID := docID[:index] seqs := make([]int64, db.msgTable.GetSingleGocMsgNum()) - minSeq := db.msgTable.GetMinSeq(index) + minSeq := db.msgTable.GetMinSeq(docIndex) for i := range seqs { seqs[i] = minSeq + int64(i) } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 851ec99c4..536827450 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -243,7 +243,14 @@ func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, li "$add": []any{ bson.M{ "$toLong": "$latest_msg_destruct_time", - }, "$msg_destruct_time"}, + }, + bson.M{ + "$multiply": []any{ + "$msg_destruct_time", + 1000, // convert to milliseconds + }, + }, + }, }, }, }, diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index c440d4442..83fefbfe6 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1091,22 +1091,148 @@ func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []i return msgDocModel[0].Msg, nil } +//func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { +// if len(seqs) == 0 { +// return nil, nil +// } +// result := make([]*model.MsgInfoModel, 0, len(seqs)) +// for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { +// res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) +// if err != nil { +// return nil, err +// } +// for i, re := range res { +// if re == nil || re.Msg == nil { +// continue +// } +// result = append(result, res[i]) +// } +// } +// return result, nil +//} + +func (m *MsgMgo) findBeforeDocSendTime(ctx context.Context, docID string, limit int64) (int64, int64, error) { + if limit == 0 { + return 0, 0, nil + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": docID, + }, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 0, + }, + }, + { + "$unwind": "$msgs", + }, + { + "$project": bson.M{ + //"_id": 0, + //"doc_id": 0, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + } + if limit > 0 { + pipeline = append(pipeline, bson.M{"$limit": limit}) + } + type Result struct { + Msgs *model.MsgInfoModel `bson:"msgs"` + } + res, err := mongoutil.Aggregate[Result](ctx, m.coll, pipeline) + if err != nil { + return 0, 0, err + } + for i := len(res) - 1; i > 0; i-- { + v := res[i] + if v.Msgs != nil && v.Msgs.Msg != nil && v.Msgs.Msg.SendTime > 0 { + return v.Msgs.Msg.Seq, v.Msgs.Msg.SendTime, nil + } + } + return 0, 0, nil +} + +func (m *MsgMgo) findBeforeSendTime(ctx context.Context, conversationID string, seq int64) (int64, int64, error) { + first := true + for i := m.model.GetDocIndex(seq); i >= 0; i-- { + limit := int64(-1) + if first { + first = false + limit = m.model.GetMsgIndex(seq) + } + docID := m.model.BuildDocIDByIndex(conversationID, i) + msgSeq, msgSendTime, err := m.findBeforeDocSendTime(ctx, docID, limit) + if err != nil { + return 0, 0, err + } + if msgSendTime > 0 { + return msgSeq, msgSendTime, nil + } + } + return 0, 0, nil +} + func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { if len(seqs) == 0 { return nil, nil } + var abnormalSeq []int64 result := make([]*model.MsgInfoModel, 0, len(seqs)) - for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { - res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) + for docID, docSeqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { + res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(docSeqs, m.model.GetMsgIndex)) if err != nil { return nil, err } + if len(res) == 0 { + abnormalSeq = append(abnormalSeq, docSeqs...) + continue + } for i, re := range res { - if re == nil || re.Msg == nil { + if re == nil || re.Msg == nil || re.Msg.SendTime == 0 { + abnormalSeq = append(abnormalSeq, docSeqs[i]) continue } result = append(result, res[i]) } } + if len(abnormalSeq) > 0 { + datautil.Sort(abnormalSeq, false) + sendTime := make(map[int64]int64) + var ( + lastSeq int64 + lastSendTime int64 + ) + for _, seq := range abnormalSeq { + if lastSendTime > 0 && lastSeq <= seq { + sendTime[seq] = lastSendTime + continue + } + msgSeq, msgSendTime, err := m.findBeforeSendTime(ctx, conversationID, seq) + if err != nil { + return nil, err + } + if msgSendTime <= 0 { + break + } + sendTime[seq] = msgSendTime + lastSeq = msgSeq + lastSendTime = msgSendTime + } + for _, seq := range abnormalSeq { + result = append(result, &model.MsgInfoModel{ + Msg: &model.MsgDataModel{ + Seq: seq, + Status: constant.MsgStatusHasDeleted, + SendTime: sendTime[seq], + }, + }) + } + } return result, nil } diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index 69113032d..6cf63bfcd 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -15,9 +15,10 @@ package model import ( + "strconv" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" - "strconv" ) const ( @@ -108,6 +109,10 @@ func (m *MsgDocModel) IsFull() bool { return m.Msg[len(m.Msg)-1].Msg != nil } +func (m *MsgDocModel) GetDocIndex(seq int64) int64 { + return (seq - 1) / singleGocMsgNum +} + func (m *MsgDocModel) GetDocID(conversationID string, seq int64) string { seqSuffix := (seq - 1) / singleGocMsgNum return m.indexGen(conversationID, seqSuffix) @@ -135,6 +140,10 @@ func (*MsgDocModel) indexGen(conversationID string, seqSuffix int64) string { return conversationID + ":" + strconv.FormatInt(seqSuffix, 10) } +func (*MsgDocModel) BuildDocIDByIndex(conversationID string, index int64) string { + return conversationID + ":" + strconv.FormatInt(index, 10) +} + func (*MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdkws.MsgData) { for _, v := range seqs { msgModel := new(sdkws.MsgData) diff --git a/tools/seq/internal/seq.go b/tools/seq/internal/seq.go index b958f1936..7e5d5598c 100644 --- a/tools/seq/internal/seq.go +++ b/tools/seq/internal/seq.go @@ -15,16 +15,17 @@ import ( "syscall" "time" + "github.com/mitchellh/mapstructure" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/runtimeenv" "github.com/redis/go-redis/v9" + "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "gopkg.in/yaml.v3" ) const ( @@ -45,13 +46,19 @@ func readConfig[T any](dir string, name string) (*T, error) { if runtimeenv.RuntimeEnvironment() == config.KUBERNETES { dir = os.Getenv(config.MountConfigFilePath) } - - data, err := os.ReadFile(filepath.Join(dir, name)) - if err != nil { + v := viper.New() + v.SetEnvPrefix(config.EnvPrefixMap[name]) + v.AutomaticEnv() + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.SetConfigFile(filepath.Join(dir, name)) + if err := v.ReadInConfig(); err != nil { return nil, err } + fn := func(config *mapstructure.DecoderConfig) { + config.TagName = "mapstructure" + } var conf T - if err := yaml.Unmarshal(data, &conf); err != nil { + if err := v.Unmarshal(&conf, fn); err != nil { return nil, err } return &conf, nil