blackteay/open-im-server
1
0
Fork 0
mirror of https://github.com/openimsdk/open-im-server.git synced 2025-04-05 20:11:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add More Multi Login Policy (#2770)

Browse Source 
* feat: multiLogin

* feat: change config
This commit is contained in:
icey-yu 2024-10-24 15:02:44 +08:00 committed by GitHub
parent a2110e416a
commit 9e8a389698
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 375 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
config/share.yml
View File

@ -13,4 +13,17 @@ rpcRegisterName:
imAdminUserID: [ imAdmin ]
# 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time
multiLoginPolicy: 1
multiLogin:
policy: 1
maxNumOneEnd: 30
customizeLoginNum:
ios: 1
android: 1
windows: 1
osx: 1
web: 1
miniWeb: 1
linux: 1
aPad: 1
iPad: 1
admin: 1

2
go.mod
View File

@ -12,7 +12,7 @@ require (
github.com/gorilla/websocket v1.5.1
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/mitchellh/mapstructure v1.5.0
github.com/openimsdk/protocol v0.0.72-alpha.41
github.com/openimsdk/protocol v0.0.72-alpha.44
github.com/openimsdk/tools v0.0.50-alpha.16
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.18.0

4
go.sum
View File

@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y=
github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI=
github.com/openimsdk/protocol v0.0.72-alpha.41 h1:SMMoTc1iu+wtRqUqmIgqPJFejLgPeauOwoJ4VVG4iMQ=
github.com/openimsdk/protocol v0.0.72-alpha.41/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8=
github.com/openimsdk/protocol v0.0.72-alpha.44 h1:TFFzkasClsYkYFKB7aZsaskO1BVJV88yshRhyxQE+/c=
github.com/openimsdk/protocol v0.0.72-alpha.44/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8=
github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc=
github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=

70
internal/msggateway/ws_server.go
View File

@ -1,17 +1,3 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package msggateway
import (
@ -212,7 +198,6 @@ func (ws *WsServer) sendUserOnlineInfoToOtherNode(ctx context.Context, client *C
if err != nil {
return err
}
wg := errgroup.Group{}
wg.SetLimit(concurrentRequest)
@ -321,8 +306,32 @@ func (ws *WsServer) KickUserConn(client *Client) error {
}
func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Client, newClient *Client) {
switch ws.msgGatewayConfig.Share.MultiLoginPolicy {
kickTokenFunc := func(kickClients []*Client) {
var kickTokens []string
ws.clients.DeleteClients(newClient.UserID, kickClients)
for _, c := range kickClients {
kickTokens = append(kickTokens, c.token)
err := c.KickOnlineMessage()
if err != nil {
log.ZWarn(c.ctx, "KickOnlineMessage", err)
}
}
ctx := mcontext.WithMustInfoCtx(
[]string{newClient.ctx.GetOperationID(), newClient.ctx.GetUserID(),
constant.PlatformIDToName(newClient.PlatformID), newClient.ctx.GetConnID()},
)
if _, err := ws.authClient.KickTokens(ctx, kickTokens); err != nil {
log.ZWarn(newClient.ctx, "kickTokens err", err)
}
}
switch ws.msgGatewayConfig.Share.MultiLogin.Policy {
case constant.DefalutNotKick:
case constant.WebAndOther:
if constant.PlatformIDToClass(newClient.PlatformID) == constant.WebPlatformStr {
return
}
fallthrough
case constant.PCAndOther:
if constant.PlatformIDToClass(newClient.PlatformID) == constant.TerminalPC {
return
@ -347,6 +356,35 @@ func (ws *WsServer) multiTerminalLoginChecker(clientOK bool, oldClients []*Clien
log.ZWarn(newClient.ctx, "InvalidateToken err", err, "userID", newClient.UserID,
"platformID", newClient.PlatformID)
}
case constant.PcMobileAndWeb:
clients, ok := ws.clients.GetAll(newClient.UserID)
if !ok {
return
}
var (
kickClients []*Client
)
for _, client := range clients {
if constant.PlatformIDToClass(client.PlatformID) == constant.PlatformIDToClass(newClient.PlatformID) {
kickClients = append(kickClients, client)
}
}
kickTokenFunc(kickClients)
case constant.SingleTerminalLogin:
clients, ok := ws.clients.GetAll(newClient.UserID)
if !ok {
return
}
var (
kickClients []*Client
)
for _, client := range clients {
kickClients = append(kickClients, client)
}
kickTokenFunc(kickClients)
case constant.Customize:
// todo
}
}

9
internal/rpc/auth/auth.go
View File

@ -65,7 +65,7 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg
redis2.NewTokenCacheModel(rdb, config.RpcConfig.TokenPolicy.Expire),
config.Share.Secret,
config.RpcConfig.TokenPolicy.Expire,
config.Share.MultiLoginPolicy,
config.Share.MultiLogin,
),
config: config,
})
@ -230,3 +230,10 @@ func (s *authServer) InvalidateToken(ctx context.Context, req *pbauth.Invalidate
}
return &pbauth.InvalidateTokenResp{}, nil
}
func (s *authServer) KickTokens(ctx context.Context, req *pbauth.KickTokensReq) (*pbauth.KickTokensResp, error) {
if err := s.authDatabase.BatchSetTokenMapByUidPid(ctx, req.Tokens); err != nil {
return nil, err
}
return &pbauth.KickTokensResp{}, nil
}

26
pkg/common/config/config.go
View File

@ -361,11 +361,29 @@ type AfterConfig struct {
}
type Share struct {
Secret string `mapstructure:"secret"`
RpcRegisterName RpcRegisterName `mapstructure:"rpcRegisterName"`
IMAdminUserID []string `mapstructure:"imAdminUserID"`
MultiLoginPolicy int `mapstructure:"multiLoginPolicy"`
Secret string `mapstructure:"secret"`
RpcRegisterName RpcRegisterName `mapstructure:"rpcRegisterName"`
IMAdminUserID []string `mapstructure:"imAdminUserID"`
MultiLogin MultiLogin `mapstructure:"multiLogin"`
}
type MultiLogin struct {
Policy int `mapstructure:"policy"`
MaxNumOneEnd int `mapstructure:"maxNumOneEnd"`
CustomizeLoginNum struct {
IOS int `mapstructure:"ios"`
Android int `mapstructure:"android"`
Windows int `mapstructure:"windows"`
OSX int `mapstructure:"osx"`
Web int `mapstructure:"web"`
MiniWeb int `mapstructure:"miniWeb"`
Linux int `mapstructure:"linux"`
APad int `mapstructure:"aPad"`
IPad int `mapstructure:"iPad"`
Admin int `mapstructure:"admin"`
} `mapstructure:"customizeLoginNum"`
}
type RpcRegisterName struct {
User string `mapstructure:"user"`
Friend string `mapstructure:"friend"`

19
pkg/common/storage/cache/cachekey/token.go vendored
View File

@ -1,6 +1,9 @@
package cachekey
import "github.com/openimsdk/protocol/constant"
import (
"github.com/openimsdk/protocol/constant"
"strings"
)
const (
UidPidToken = "UID_PID_TOKEN_STATUS:"
@ -9,3 +12,17 @@ const (
func GetTokenKey(userID string, platformID int) string {
return UidPidToken + userID + ":" + constant.PlatformIDToName(platformID)
}
func GetAllPlatformTokenKey(userID string) []string {
res := make([]string, len(constant.PlatformID2Name))
for k := range constant.PlatformID2Name {
res[k-1] = GetTokenKey(userID, k)
}
return res
}
func GetPlatformIDByTokenKey(key string) int {
splitKey := strings.Split(key, ":")
platform := splitKey[len(splitKey)-1]
return constant.PlatformNameToID(platform)
}

64
pkg/common/storage/cache/redis/token.go vendored
View File

@ -1,17 +1,3 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package redis
import (
@ -21,6 +7,7 @@ import (
"github.com/openimsdk/tools/errs"
"github.com/redis/go-redis/v9"
"strconv"
"sync"
"time"
)
@ -67,6 +54,43 @@ func (c *tokenCache) GetTokensWithoutError(ctx context.Context, userID string, p
return mm, nil
}
func (c *tokenCache) GetAllTokensWithoutError(ctx context.Context, userID string) (map[int]map[string]int, error) {
var (
res = make(map[int]map[string]int)
resLock = sync.Mutex{}
)
keys := cachekey.GetAllPlatformTokenKey(userID)
if err := ProcessKeysBySlot(ctx, c.rdb, keys, func(ctx context.Context, slot int64, keys []string) error {
pipe := c.rdb.Pipeline()
mapRes := make([]*redis.MapStringStringCmd, len(keys))
for i, key := range keys {
mapRes[i] = pipe.HGetAll(ctx, key)
}
_, err := pipe.Exec(ctx)
if err != nil {
return err
}
for i, m := range mapRes {
mm := make(map[string]int)
for k, v := range m.Val() {
state, err := strconv.Atoi(v)
if err != nil {
return errs.WrapMsg(err, "redis token value is not int", "value", v, "userID", userID)
}
mm[k] = state
}
resLock.Lock()
res[cachekey.GetPlatformIDByTokenKey(keys[i])] = mm
resLock.Unlock()
}
return nil
}); err != nil {
return nil, err
}
return res, nil
}
func (c *tokenCache) SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error {
mm := make(map[string]any)
for k, v := range m {
@ -75,6 +99,18 @@ func (c *tokenCache) SetTokenMapByUidPid(ctx context.Context, userID string, pla
return errs.Wrap(c.rdb.HSet(ctx, cachekey.GetTokenKey(userID, platformID), mm).Err())
}
func (c *tokenCache) BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]int) error {
pipe := c.rdb.Pipeline()
for k, v := range tokens {
pipe.HSet(ctx, k, v)
}
_, err := pipe.Exec(ctx)
if err != nil {
return errs.Wrap(err)
}
return nil
}
func (c *tokenCache) DeleteTokenByUidPid(ctx context.Context, userID string, platformID int, fields []string) error {
return errs.Wrap(c.rdb.HDel(ctx, cachekey.GetTokenKey(userID, platformID), fields...).Err())
}

2
pkg/common/storage/cache/token.go vendored
View File

@ -9,6 +9,8 @@ type TokenModel interface {
// SetTokenFlagEx set token and flag with expire time
SetTokenFlagEx(ctx context.Context, userID string, platformID int, token string, flag int) error
GetTokensWithoutError(ctx context.Context, userID string, platformID int) (map[string]int, error)
GetAllTokensWithoutError(ctx context.Context, userID string) (map[int]map[string]int, error)
SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error
BatchSetTokenMapByUidPid(ctx context.Context, tokens map[string]map[string]int) error
DeleteTokenByUidPid(ctx context.Context, userID string, platformID int, fields []string) error
}

251
pkg/common/storage/controller/auth.go
View File

@ -1,21 +1,9 @@
// Copyright © 2023 OpenIM. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controller
import (
"context"
"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey"
"github.com/openimsdk/tools/log"
"github.com/golang-jwt/jwt/v4"
@ -32,18 +20,41 @@ type AuthDatabase interface {
// Create token
CreateToken(ctx context.Context, userID string, platformID int) (string, error)
BatchSetTokenMapByUidPid(ctx context.Context, tokens []string) error
SetTokenMapByUidPid(ctx context.Context, userID string, platformID int, m map[string]int) error
}
type authDatabase struct {
cache cache.TokenModel
accessSecret string
accessExpire int64
multiLoginPolicy int
type multiLoginConfig struct {
Policy int
MaxNumOneEnd int
CustomizeLoginNum map[int]int
}
func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64, policy int) AuthDatabase {
return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, multiLoginPolicy: policy}
type authDatabase struct {
cache cache.TokenModel
accessSecret string
accessExpire int64
multiLogin multiLoginConfig
}
func NewAuthDatabase(cache cache.TokenModel, accessSecret string, accessExpire int64, multiLogin config.MultiLogin) AuthDatabase {
return &authDatabase{cache: cache, accessSecret: accessSecret, accessExpire: accessExpire, multiLogin: multiLoginConfig{
Policy: multiLogin.Policy,
MaxNumOneEnd: multiLogin.MaxNumOneEnd,
CustomizeLoginNum: map[int]int{
constant.IOSPlatformID: multiLogin.CustomizeLoginNum.IOS,
constant.AndroidPlatformID: multiLogin.CustomizeLoginNum.Android,
constant.WindowsPlatformID: multiLogin.CustomizeLoginNum.Windows,
constant.OSXPlatformID: multiLogin.CustomizeLoginNum.OSX,
constant.WebPlatformID: multiLogin.CustomizeLoginNum.Web,
constant.MiniWebPlatformID: multiLogin.CustomizeLoginNum.MiniWeb,
constant.LinuxPlatformID: multiLogin.CustomizeLoginNum.Linux,
constant.AndroidPadPlatformID: multiLogin.CustomizeLoginNum.APad,
constant.IPadPlatformID: multiLogin.CustomizeLoginNum.IPad,
constant.AdminPlatformID: multiLogin.CustomizeLoginNum.Admin,
},
}}
}
// If the result is empty.
@ -55,22 +66,38 @@ func (a *authDatabase) SetTokenMapByUidPid(ctx context.Context, userID string, p
return a.cache.SetTokenMapByUidPid(ctx, userID, platformID, m)
}
func (a *authDatabase) BatchSetTokenMapByUidPid(ctx context.Context, tokens []string) error {
setMap := make(map[string]map[string]int)
for _, token := range tokens {
claims, err := tokenverify.GetClaimFromToken(token, authverify.Secret(a.accessSecret))
key := cachekey.GetTokenKey(claims.UserID, claims.PlatformID)
if err != nil {
continue
} else {
if v, ok := setMap[key]; ok {
v[token] = constant.KickedToken
} else {
setMap[key] = map[string]int{
token: constant.KickedToken,
}
}
}
}
if err := a.cache.BatchSetTokenMapByUidPid(ctx, setMap); err != nil {
return err
}
return nil
}
// Create Token.
func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformID int) (string, error) {
// todo: get all platform token
tokens, err := a.cache.GetTokensWithoutError(ctx, userID, platformID)
tokens, err := a.cache.GetAllTokensWithoutError(ctx, userID)
if err != nil {
return "", err
}
var deleteTokenKey []string
var kickedTokenKey []string
for k, v := range tokens {
t, err := tokenverify.GetClaimFromToken(k, authverify.Secret(a.accessSecret))
if err != nil || v != constant.NormalToken {
deleteTokenKey = append(deleteTokenKey, k)
} else if a.checkKickToken(ctx, platformID, t) {
kickedTokenKey = append(kickedTokenKey, k)
}
deleteTokenKey, kickedTokenKey, err := a.checkToken(ctx, tokens, platformID)
if err != nil {
return "", err
}
if len(deleteTokenKey) != 0 {
err = a.cache.DeleteTokenByUidPid(ctx, userID, platformID, deleteTokenKey)
@ -78,16 +105,6 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI
return "", err
}
}
const adminTokenMaxNum = 30
if platformID == constant.AdminPlatformID {
if len(kickedTokenKey) > adminTokenMaxNum {
kickedTokenKey = kickedTokenKey[:len(kickedTokenKey)-adminTokenMaxNum]
} else {
kickedTokenKey = nil
}
}
if len(kickedTokenKey) != 0 {
for _, k := range kickedTokenKey {
err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken)
@ -111,22 +128,140 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI
return tokenString, nil
}
func (a *authDatabase) checkKickToken(ctx context.Context, platformID int, token *tokenverify.Claims) bool {
switch a.multiLoginPolicy {
case constant.DefalutNotKick:
return false
case constant.PCAndOther:
if constant.PlatformIDToClass(platformID) == constant.TerminalPC ||
constant.PlatformIDToClass(token.PlatformID) == constant.TerminalPC {
return false
func (a *authDatabase) checkToken(ctx context.Context, tokens map[int]map[string]int, platformID int) ([]string, []string, error) {
// todo: Move the logic for handling old data to another location.
var (
loginTokenMap = make(map[int][]string) // The length of the value of the map must be greater than 0
deleteToken = make([]string, 0)
kickToken = make([]string, 0)
adminToken = make([]string, 0)
unkickTerminal = ""
)
for plfID, tks := range tokens {
for k, v := range tks {
_, err := tokenverify.GetClaimFromToken(k, authverify.Secret(a.accessSecret))
if err != nil || v != constant.NormalToken {
deleteToken = append(deleteToken, k)
} else {
if plfID != constant.AdminPlatformID {
loginTokenMap[plfID] = append(loginTokenMap[plfID], k)
} else {
adminToken = append(adminToken, k)
}
}
}
return true
case constant.AllLoginButSameTermKick:
if platformID == token.PlatformID {
return true
}
return false
default:
return false
}
switch a.multiLogin.Policy {
case constant.DefalutNotKick:
for plt, ts := range loginTokenMap {
l := len(ts)
if platformID == plt {
l++
}
limit := a.multiLogin.MaxNumOneEnd
if l > limit {
kickToken = append(kickToken, ts[:l-limit]...)
}
}
case constant.AllLoginButSameTermKick:
for plt, ts := range loginTokenMap {
kickToken = append(kickToken, ts[:len(ts)-1]...)
if plt == platformID {
kickToken = append(kickToken, ts[len(ts)-1])
}
}
case constant.SingleTerminalLogin:
for _, ts := range loginTokenMap {
kickToken = append(kickToken, ts...)
}
case constant.WebAndOther:
unkickTerminal = constant.WebPlatformStr
fallthrough
case constant.PCAndOther:
if unkickTerminal == "" {
unkickTerminal = constant.TerminalPC
}
if constant.PlatformIDToClass(platformID) != unkickTerminal {
for plt, ts := range loginTokenMap {
if constant.PlatformIDToClass(plt) != unkickTerminal {
kickToken = append(kickToken, ts...)
}
}
} else {
var (
preKick []string
isReserve = true
)
for plt, ts := range loginTokenMap {
if constant.PlatformIDToClass(plt) != unkickTerminal {
// Keep a token from another end
if isReserve {
isReserve = false
kickToken = append(kickToken, ts[:len(ts)-1]...)
preKick = append(preKick, ts[len(ts)-1])
continue
} else {
// Prioritize keeping Android
if plt == constant.AndroidPlatformID {
kickToken = append(kickToken, preKick...)
kickToken = append(kickToken, ts[:len(ts)-1]...)
} else {
kickToken = append(kickToken, ts...)
}
}
}
}
}
case constant.PcMobileAndWeb:
var (
reserved = make(map[string]bool)
)
for plt, ts := range loginTokenMap {
if constant.PlatformIDToClass(plt) == constant.PlatformIDToClass(platformID) {
kickToken = append(kickToken, ts...)
} else {
if !reserved[constant.PlatformIDToClass(plt)] {
reserved[constant.PlatformIDToClass(plt)] = true
kickToken = append(kickToken, ts[:len(ts)-1]...)
continue
} else {
kickToken = append(kickToken, ts...)
}
}
}
case constant.Customize:
if a.multiLogin.CustomizeLoginNum[platformID] <= 0 {
return nil, nil, errs.New("Do not allow login on this end").Wrap()
}
for plt, ts := range loginTokenMap {
l := len(ts)
if platformID == plt {
l++
}
// a.multiLogin.CustomizeLoginNum[platformID] must > 0
limit := min(a.multiLogin.CustomizeLoginNum[plt], a.multiLogin.MaxNumOneEnd)
if l > limit {
kickToken = append(kickToken, ts[:l-limit]...)
}
}
default:
return nil, nil, errs.New("unknown multiLogin policy").Wrap()
}
var adminTokenMaxNum = a.multiLogin.MaxNumOneEnd
if a.multiLogin.Policy == constant.Customize {
adminTokenMaxNum = a.multiLogin.CustomizeLoginNum[constant.AdminPlatformID]
}
l := len(adminToken)
if platformID == constant.AdminPlatformID {
l++
}
if l > adminTokenMaxNum {
kickToken = append(kickToken, adminToken[:l-adminTokenMaxNum]...)
}
return deleteToken, kickToken, nil
}

11
pkg/rpcclient/auth.go
View File

@ -61,3 +61,14 @@ func (a *Auth) InvalidateToken(ctx context.Context, preservedToken, userID strin
}
return resp, err
}
func (a *Auth) KickTokens(ctx context.Context, tokens []string) (*auth.KickTokensResp, error) {
req := auth.KickTokensReq{
Tokens: tokens,
}
resp, err := a.Client.KickTokens(ctx, &req)
if err != nil {
return nil, err
}
return resp, err
}