Files
zeekay 8feb24ea1f One name for this fork, and it is ours
The server module named github.com/hanzoai/team/server/v8 — a path that
resolves to the retired workspace's repository, not this one — while
server/public still published github.com/mattermost/mattermost/server/public.
Both are github.com/hanzoteam/server/... now, which is where this code
actually lives.

That second half is what made the binary unbootable. Upstream's public path
stayed claimed by upstream, mattermost-plugin-ai imported it, and the build
linked public/model twice; public/plugin's gob.Register then ran twice under
one name and panicked at init, before main. Reverting the rename traded the
brand for a working binary. Rebranding the plugin alongside it needs no such
trade: the agents fork is github.com/hanzoteam/agents/v2 against our public
module, so no module named mattermost/mattermost is in the graph at all.

Measured: `go list -deps ./cmd/mattermost` names exactly one public/model, and
it is ours.
2026-08-28 23:35:36 -07:00

1322 lines
44 KiB
Go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sharedchannel
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
"github.com/wiggin77/merror"
"github.com/hanzoteam/server/server/public/model"
"github.com/hanzoteam/server/server/public/shared/mlog"
"github.com/hanzoteam/server/server/public/shared/request"
"github.com/hanzoteam/server/server/v8/channels/store"
"github.com/hanzoteam/server/server/v8/platform/services/remotecluster"
)
type sendSyncMsgResultFunc func(syncResp model.SyncResponse, err error)
type attachment struct {
fi *model.FileInfo
post *model.Post
}
type syncData struct {
task syncTask
rc *model.RemoteCluster
scr *model.SharedChannelRemote
users map[string]*model.User
profileImages map[string]*model.User
posts []*model.Post
reactions []*model.Reaction
acknowledgements []*model.PostAcknowledgement
statuses []*model.Status
attachments []attachment
mentionTransforms map[string]string
membershipChanges []*model.MembershipChangeMsg
resultNextMembershipCursor int64
resultRepeat bool
resultNextCursor model.GetPostsSinceForSyncCursor
GlobalUserSyncLastTimestamp int64
}
func newSyncData(task syncTask, rc *model.RemoteCluster, scr *model.SharedChannelRemote) *syncData {
return &syncData{
task: task,
rc: rc,
scr: scr,
users: make(map[string]*model.User),
profileImages: make(map[string]*model.User),
mentionTransforms: make(map[string]string),
resultNextCursor: model.GetPostsSinceForSyncCursor{
LastPostUpdateAt: scr.LastPostUpdateAt, LastPostUpdateID: scr.LastPostUpdateID,
LastPostCreateAt: scr.LastPostCreateAt, LastPostCreateID: scr.LastPostCreateID,
},
}
}
func (sd *syncData) isEmpty() bool {
return len(sd.users) == 0 && len(sd.profileImages) == 0 && len(sd.posts) == 0 && len(sd.reactions) == 0 && len(sd.acknowledgements) == 0 && len(sd.attachments) == 0 && len(sd.membershipChanges) == 0
}
func (sd *syncData) isCursorChanged() bool {
if sd.resultNextCursor.IsEmpty() {
return false
}
return sd.scr.LastPostCreateAt != sd.resultNextCursor.LastPostCreateAt || sd.scr.LastPostCreateID != sd.resultNextCursor.LastPostCreateID ||
sd.scr.LastPostUpdateAt != sd.resultNextCursor.LastPostUpdateAt || sd.scr.LastPostUpdateID != sd.resultNextCursor.LastPostUpdateID
}
func (sd *syncData) setDataFromMsg(msg *model.SyncMsg) {
sd.users = msg.Users
sd.posts = msg.Posts
sd.reactions = msg.Reactions
sd.acknowledgements = msg.Acknowledgements
sd.statuses = msg.Statuses
}
// syncForRemote updates a remote cluster with any new posts/reactions for a specific
// channel. If many changes are found, only the oldest X changes are sent and the channel
// is re-added to the task map. This ensures no channels are starved for updates even if some
// channels are very active.
// Returning an error forces a retry on the task.
func (scs *Service) syncForRemote(task syncTask, rc *model.RemoteCluster) error {
// Empty channelID indicates a global user sync task
// Normal syncTasks always include a valid channelID
if task.channelID == "" {
return scs.syncAllUsers(rc)
}
rcs := scs.server.GetRemoteClusterService()
if rcs == nil {
return fmt.Errorf("cannot update remote cluster %s for channel id %s; Remote Cluster Service not enabled", rc.Name, task.channelID)
}
metrics := scs.server.GetMetrics()
start := time.Now()
var metricsRecorded bool
defer func() {
if !metricsRecorded && metrics != nil {
metrics.IncrementSharedChannelsSyncCounter(rc.RemoteId)
metrics.ObserveSharedChannelsSyncCollectionDuration(rc.RemoteId, time.Since(start).Seconds())
metricsRecorded = true
}
}()
scr, err := scs.server.GetStore().SharedChannel().GetRemoteByIds(task.channelID, rc.RemoteId)
if isNotFoundError(err) && rc.IsOptionFlagSet(model.BitflagOptionAutoInvited) {
// if SharedChannelRemote not found and remote has autoinvite flag, create a scr for it, thus inviting the remote.
scr = &model.SharedChannelRemote{
Id: model.NewId(),
ChannelId: task.channelID,
CreatorId: rc.CreatorId,
IsInviteAccepted: true,
IsInviteConfirmed: true,
RemoteId: rc.RemoteId,
LastPostCreateAt: model.GetMillis(),
LastPostUpdateAt: model.GetMillis(),
LastMembersSyncAt: 0,
}
if scr, err = scs.server.GetStore().SharedChannel().SaveRemote(scr); err != nil {
return fmt.Errorf("cannot auto-create shared channel remote (channel_id=%s, remote_id=%s): %w", task.channelID, rc.RemoteId, err)
}
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Auto-invited remote to channel (BitflagOptionAutoInvited)",
mlog.String("remote", rc.DisplayName),
mlog.String("channel_id", task.channelID),
)
} else if err == nil && scr.DeleteAt != 0 {
// if SharedChannelRemote is deleted, regardless of the autoinvite flag, do nothing
return nil
} else if err != nil {
return err
}
sd := newSyncData(task, rc, scr)
// if this is retrying a failed msg, just send it again.
if task.retryMsg != nil {
sd.setDataFromMsg(task.retryMsg)
return scs.sendSyncData(sd)
}
// if this has an already existing msg, just send it right away
if task.existingMsg != nil {
sd.setDataFromMsg(task.existingMsg)
return scs.sendSyncData(sd)
}
// if we don't have a channelID at this point, we cannot fetch new
// data from the database
if task.channelID == "" {
return fmt.Errorf("task doesn't have prefetched data nor a channel ID set")
}
// schedule another sync if the repeat flag is set at some point.
defer func(rpt *bool) {
if *rpt {
scs.addTask(newSyncTask(task.channelID, task.userID, task.remoteID, nil, nil))
}
}(&sd.resultRepeat)
// fetch new posts or retry post.
if err := scs.fetchPostsForSync(sd); err != nil {
return fmt.Errorf("cannot fetch posts for sync %v: %w", sd, err)
}
// fetch membership changes from ChannelMemberHistory
if err := scs.fetchMembershipsForSync(sd); err != nil {
return fmt.Errorf("cannot fetch memberships for sync %v: %w", sd, err)
}
if !rc.IsOnline() {
if len(sd.posts) != 0 {
scs.notifyRemoteOffline(sd.posts, rc)
}
sd.resultRepeat = false
return nil
}
// fetch users that have updated their user profile or image.
if err := scs.fetchUsersForSync(sd); err != nil {
return fmt.Errorf("cannot fetch users for sync %v: %w", sd, err)
}
// fetch reactions for posts
if err := scs.fetchReactionsForSync(sd); err != nil {
return fmt.Errorf("cannot fetch reactions for sync %v: %w", sd, err)
}
// fetch acknowledgements for posts
if err := scs.fetchAcknowledgementsForSync(sd); err != nil {
return fmt.Errorf("cannot fetch acknowledgements for sync %v: %w", sd, err)
}
// fetch users associated with posts & reactions
if err := scs.fetchPostUsersForSync(sd); err != nil {
return fmt.Errorf("cannot fetch post users for sync %v: %w", sd, err)
}
// filter out any posts that don't need to be sent.
scs.filterPostsForSync(sd)
// fetch attachments for posts
if err := scs.fetchPostAttachmentsForSync(sd); err != nil {
return fmt.Errorf("cannot fetch post attachments for sync %v: %w", sd, err)
}
if sd.isEmpty() {
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Not sending sync data; everything filtered out",
mlog.String("remote", rc.DisplayName),
mlog.String("channel_id", task.channelID),
mlog.Bool("repeat", sd.resultRepeat),
)
if sd.isCursorChanged() {
scs.updateCursorForRemote(sd.scr.Id, sd.rc, sd.resultNextCursor)
}
return nil
}
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Sending sync data",
mlog.String("remote", rc.DisplayName),
mlog.String("channel_id", task.channelID),
mlog.Bool("repeat", sd.resultRepeat),
mlog.Int("users", len(sd.users)),
mlog.Int("images", len(sd.profileImages)),
mlog.Int("posts", len(sd.posts)),
mlog.Int("reactions", len(sd.reactions)),
mlog.Int("acknowledgements", len(sd.acknowledgements)),
mlog.Int("attachments", len(sd.attachments)),
)
if !metricsRecorded && metrics != nil {
metrics.IncrementSharedChannelsSyncCounter(rc.RemoteId)
metrics.ObserveSharedChannelsSyncCollectionDuration(rc.RemoteId, time.Since(start).Seconds())
metricsRecorded = true
}
return scs.sendSyncData(sd)
}
// fetchUsersForSync populates the sync data with any channel users who updated their user profile
// since the last sync.
func (scs *Service) fetchUsersForSync(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncCollectionStepDuration(sd.rc.RemoteId, "Users", time.Since(start).Seconds())
}
}()
filter := model.GetUsersForSyncFilter{
ChannelID: sd.task.channelID,
Limit: MaxUsersPerSync,
}
users, err := scs.server.GetStore().SharedChannel().GetUsersForSync(filter)
if err != nil {
return err
}
// Don't sync users back to the remote cluster they originated from
for _, u := range users {
if u.GetRemoteID() != sd.rc.RemoteId {
sd.users[u.Id] = u
}
}
filter.CheckProfileImage = true
usersImage, err := scs.server.GetStore().SharedChannel().GetUsersForSync(filter)
if err != nil {
return err
}
for _, u := range usersImage {
if u.GetRemoteID() != sd.rc.RemoteId {
sd.profileImages[u.Id] = u
}
}
return nil
}
// fetchPostsForSync populates the sync data with any new or edited posts since the last sync.
func (scs *Service) fetchPostsForSync(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncCollectionStepDuration(sd.rc.RemoteId, "Posts", time.Since(start).Seconds())
}
}()
options := model.GetPostsSinceForSyncOptions{
ChannelId: sd.task.channelID,
IncludeDeleted: true,
SinceCreateAt: true,
ExcludeChannelMetadataSystemPosts: true,
ExcludedPostTypes: []string{model.PostTypeCard},
}
cursor := model.GetPostsSinceForSyncCursor{
LastPostUpdateAt: sd.scr.LastPostUpdateAt,
LastPostUpdateID: sd.scr.LastPostUpdateID,
LastPostCreateAt: sd.scr.LastPostCreateAt,
LastPostCreateID: sd.scr.LastPostCreateID,
}
maxPostsPerSync := *scs.server.Config().ConnectedWorkspacesSettings.MaxPostsPerSync
// Fetch all newly created posts first. This is to ensure that post order is preserved for sync targets
// that cannot set the CreateAt timestamp for incoming posts (e.g. MS Teams). If we simply used UpdateAt
// then posts could get out of order. For example: p1 created, p2 created, p1 updated... sync'ing on UpdateAt
// would order the posts p2, p1.
posts, nextCursor, err := scs.server.GetStore().Post().GetPostsSinceForSync(options, cursor, maxPostsPerSync)
if err != nil {
return fmt.Errorf("could not fetch new posts for sync: %w", err)
}
count := len(posts)
sd.posts = appendPosts(sd.posts, posts, scs.server.GetStore().Post(), cursor.LastPostCreateAt, scs.server.Log())
cache := postsSliceToMap(posts)
// Fill remaining batch capacity with updated posts.
if len(posts) < maxPostsPerSync {
options.SinceCreateAt = false
// use 'nextcursor' as it has the correct xxxUpdateAt values, and the updsted xxxCreateAt values.
posts, nextCursor, err = scs.server.GetStore().Post().GetPostsSinceForSync(options, nextCursor, maxPostsPerSync-len(posts))
if err != nil {
return fmt.Errorf("could not fetch modified posts for sync: %w", err)
}
posts = reducePostsSliceInCache(posts, cache)
count += len(posts)
sd.posts = appendPosts(sd.posts, posts, scs.server.GetStore().Post(), cursor.LastPostUpdateAt, scs.server.Log())
}
// Shared-channel state posts are never synced to remotes. Drop them before
// PreparePostForClient and before reaction/ack/post-user collection so we do not query stores
// for posts that will be removed later. Cursor is still derived from GetPostsSinceForSync above,
// which includes these rows, so sync cursors advance the same as before.
stripSharedChannelStatePostsForSync(sd)
// Populate metadata for all posts before syncing
for i, post := range sd.posts {
if post != nil {
sd.posts[i] = scs.app.PreparePostForClient(request.EmptyContext(scs.server.Log()), post, &model.PreparePostForClientOpts{IncludePriority: true})
}
}
sd.resultNextCursor = nextCursor
sd.resultRepeat = count >= maxPostsPerSync
return nil
}
// fetchMembershipsForSync populates the sync data with membership changes from ChannelMemberHistory.
func (scs *Service) fetchMembershipsForSync(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncCollectionStepDuration(sd.rc.RemoteId, "Memberships", time.Since(start).Seconds())
}
}()
cursor := sd.scr.LastMembersSyncAt
limit := scs.GetMemberSyncBatchSize()
histories, err := scs.server.GetStore().ChannelMemberHistory().GetMembershipChanges(sd.task.channelID, cursor, limit)
if err != nil {
return fmt.Errorf("could not fetch membership changes for sync: %w", err)
}
if len(histories) == 0 {
return nil
}
// Deduplicate by user — for users with multiple events (join/leave/rejoin cycles),
// resolve to the most recent event to determine current state.
type userState struct {
isAdd bool
eventTime int64 // max of JoinTime/LeaveTime for this user
}
byUser := make(map[string]*userState)
var maxCursor int64
for _, h := range histories {
// Track the max event time for the next cursor
if h.JoinTime > maxCursor {
maxCursor = h.JoinTime
}
if h.LeaveTime != nil && *h.LeaveTime > maxCursor {
maxCursor = *h.LeaveTime
}
// Determine this event's effective time and whether it represents a join or leave
var eventTime int64
var isAdd bool
if h.LeaveTime == nil || h.JoinTime > *h.LeaveTime {
// User is currently a member (no leave, or rejoined after leaving)
isAdd = true
eventTime = h.JoinTime
} else {
// User left
isAdd = false
eventTime = *h.LeaveTime
}
// Keep only the most recent state per user
if existing, ok := byUser[h.UserId]; !ok || eventTime > existing.eventTime {
byUser[h.UserId] = &userState{isAdd: isAdd, eventTime: eventTime}
}
}
// Build MembershipChangeMsg entries
for userID, state := range byUser {
user, userErr := scs.server.GetStore().User().Get(request.EmptyContext(scs.server.Log()), userID)
if userErr != nil {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Failed to get user for membership sync",
mlog.String("user_id", userID),
mlog.String("channel_id", sd.task.channelID),
mlog.Err(userErr),
)
continue
}
// Skip users that originated from the target remote — they are local
// on the receiver and their membership is managed there.
if user.GetRemoteID() == sd.rc.RemoteId {
continue
}
if state.isAdd {
sd.membershipChanges = append(sd.membershipChanges, &model.MembershipChangeMsg{
ChannelId: sd.task.channelID,
UserId: userID,
IsAdd: true,
ChangeTime: state.eventTime,
})
doSync, _, syncErr := scs.shouldUserSync(user, sd.task.channelID, sd.rc)
if syncErr == nil && doSync {
sd.users[user.Id] = user
}
} else {
sd.membershipChanges = append(sd.membershipChanges, &model.MembershipChangeMsg{
ChannelId: sd.task.channelID,
UserId: userID,
IsAdd: false,
ChangeTime: state.eventTime,
})
}
}
sd.resultNextMembershipCursor = maxCursor
// If we hit the limit, there may be more data to fetch
if len(histories) >= limit {
sd.resultRepeat = true
}
return nil
}
// sendMembershipSyncData sends the collected membership changes to the remote cluster.
func (scs *Service) sendMembershipSyncData(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncSendStepDuration(sd.rc.RemoteId, "Memberships", time.Since(start).Seconds())
}
}()
msg := model.NewSyncMsg(sd.task.channelID)
msg.MembershipChanges = sd.membershipChanges
// Include only user profiles for users referenced by membership adds,
// not the full sd.users map which may contain post authors and other users.
memberUsers := make(map[string]*model.User)
for _, mc := range sd.membershipChanges {
if mc.IsAdd {
if u, ok := sd.users[mc.UserId]; ok {
memberUsers[mc.UserId] = u
}
}
}
msg.Users = memberUsers
return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
if len(syncResp.MembershipErrors) != 0 {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Response indicates error for membership(s) sync",
mlog.String("channel_id", sd.task.channelID),
mlog.String("remote_id", sd.rc.RemoteId),
mlog.Array("membership_errors", syncResp.MembershipErrors),
)
}
// Update the membership cursor on success
if errResp == nil && sd.resultNextMembershipCursor > 0 {
if err := scs.updateMembershipSyncCursor(sd.task.channelID, sd.rc.RemoteId, sd.resultNextMembershipCursor); err != nil {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceError, "Failed to update membership sync cursor",
mlog.String("channel_id", sd.task.channelID),
mlog.String("remote_id", sd.rc.RemoteId),
mlog.Err(err),
)
}
}
})
}
func appendPosts(dest []*model.Post, posts []*model.Post, postStore store.PostStore, timestamp int64, logger mlog.LoggerIFace) []*model.Post {
// Append the posts individually, checking for root posts that might appear later in the list.
// This is due to the UpdateAt collision handling algorithm where the order of posts is not based
// on UpdateAt or CreateAt when the posts have the same UpdateAt value. Here we are guarding
// against a root post with the same UpdateAt (and probably the same CreateAt) appearing later
// in the list and must be sync'd before the child post. This is an edge case that likely only
// happens during load testing or bulk imports.
for _, p := range posts {
if p.RootId != "" {
root, err := postStore.GetSingle(request.EmptyContext(logger), p.RootId, true)
if err == nil {
if (root.CreateAt >= timestamp || root.UpdateAt >= timestamp) && !containsPost(dest, root) {
dest = append(dest, root)
}
}
}
dest = append(dest, p)
}
return dest
}
func containsPost(posts []*model.Post, post *model.Post) bool {
for _, p := range posts {
if p.Id == post.Id {
return true
}
}
return false
}
// fetchReactionsForSync populates the sync data with any new reactions since the last sync.
func (scs *Service) fetchReactionsForSync(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncCollectionStepDuration(sd.rc.RemoteId, "Reactions", time.Since(start).Seconds())
}
}()
merr := merror.New()
for _, post := range sd.posts {
// any reactions originating from the remote cluster are filtered out
reactions, err := scs.server.GetStore().Reaction().GetForPostSince(post.Id, sd.scr.LastPostUpdateAt, sd.rc.RemoteId, true)
if err != nil {
merr.Append(fmt.Errorf("could not get reactions for post %s: %w", post.Id, err))
continue
}
sd.reactions = append(sd.reactions, reactions...)
}
return merr.ErrorOrNil()
}
// fetchAcknowledgementsForSync populates the sync data with any new acknowledgements since the last sync.
func (scs *Service) fetchAcknowledgementsForSync(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncCollectionStepDuration(sd.rc.RemoteId, "Acknowledgements", time.Since(start).Seconds())
}
}()
merr := merror.New()
for _, post := range sd.posts {
// any acknowledgements originating from the remote cluster are filtered out
acknowledgements, err := scs.server.GetStore().PostAcknowledgement().GetForPostSince(post.Id, sd.scr.LastPostUpdateAt, sd.rc.RemoteId, true)
if err != nil {
merr.Append(fmt.Errorf("could not get acknowledgements for post %s: %w", post.Id, err))
continue
}
sd.acknowledgements = append(sd.acknowledgements, acknowledgements...)
}
return merr.ErrorOrNil()
}
// fetchPostUsersForSync populates the sync data with all users associated with posts.
func (scs *Service) fetchPostUsersForSync(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncCollectionStepDuration(sd.rc.RemoteId, "PostUsers", time.Since(start).Seconds())
}
}()
sc, err := scs.server.GetStore().SharedChannel().Get(sd.task.channelID)
if err != nil {
return fmt.Errorf("cannot determine teamID: %w", err)
}
type p2mm struct {
post *model.Post
mentionMap model.UserMentionMap
}
userIDs := make(map[string]p2mm)
for _, reaction := range sd.reactions {
userIDs[reaction.UserId] = p2mm{}
}
for _, acknowledgement := range sd.acknowledgements {
userIDs[acknowledgement.UserId] = p2mm{}
}
for _, post := range sd.posts {
// get mentions and users for each mention
mentionMap := scs.app.MentionsToTeamMembers(request.EmptyContext(scs.server.Log()), post.Message, sc.TeamId)
// Skip notifications for remote users unless mentioned with @username:remote format
for mention, userID := range mentionMap {
user, err := scs.server.GetStore().User().Get(request.EmptyContext(scs.server.Log()), userID)
if err != nil {
continue
}
// Skip remote users unless mention contains a colon (@username:remote)
if user.IsRemote() && !strings.Contains(mention, ":") {
continue
}
}
// add author with post and mentionMap so transformations can be applied
userIDs[post.UserId] = p2mm{
post: post,
mentionMap: mentionMap,
}
// Add all mentioned users
for _, userID := range mentionMap {
userIDs[userID] = p2mm{
post: post,
mentionMap: mentionMap,
}
}
}
merr := merror.New()
for userID, v := range userIDs {
user, err := scs.server.GetStore().User().Get(request.EmptyContext(scs.server.Log()), userID)
if err != nil {
merr.Append(fmt.Errorf("could not get user %s: %w", userID, err))
continue
}
sync, syncImage, err2 := scs.shouldUserSync(user, sd.task.channelID, sd.rc)
if err2 != nil {
merr.Append(fmt.Errorf("could not check should sync user %s: %w", userID, err2))
continue
}
if sync {
sd.users[user.Id] = user
}
if syncImage {
sd.profileImages[user.Id] = user
}
// Collect mention transforms for all mentioned users
if v.mentionMap != nil {
for mention, mentionUserID := range v.mentionMap {
if mentionUserID == userID {
// Always add the mention transform - let receiver decide how to display
// The sender should NOT modify the message, only provide the mapping
sd.mentionTransforms[mention] = userID
}
}
}
}
return merr.ErrorOrNil()
}
// fetchPostAttachmentsForSync populates the sync data with any file attachments for new posts.
func (scs *Service) fetchPostAttachmentsForSync(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncCollectionStepDuration(sd.rc.RemoteId, "Attachments", time.Since(start).Seconds())
}
}()
merr := merror.New()
for _, post := range sd.posts {
fis, err := scs.server.GetStore().FileInfo().GetForPost(post.Id, false, true, true)
if err != nil {
merr.Append(fmt.Errorf("could not get file attachment info for post %s: %w", post.Id, err))
continue
}
for _, fi := range fis {
if scs.shouldSyncAttachment(fi, sd.rc) {
sd.attachments = append(sd.attachments, attachment{fi: fi, post: post})
}
}
}
return merr.ErrorOrNil()
}
func stripSharedChannelStatePostsForSync(sd *syncData) {
if len(sd.posts) == 0 {
return
}
out := make([]*model.Post, 0, len(sd.posts))
for _, p := range sd.posts {
if p != nil && p.Type == model.PostTypeSharedChannelState {
continue
}
out = append(out, p)
}
sd.posts = out
}
// filterPostsForSync removes any posts that do not need to sync.
func (scs *Service) filterPostsForSync(sd *syncData) {
filtered := make([]*model.Post, 0, len(sd.posts))
for _, p := range sd.posts {
// Don't resend an existing post where only the reactions changed.
// Posts we must send:
// - new posts (EditAt == 0)
// - edited posts (EditAt >= LastPostUpdateAt)
// - deleted posts (DeleteAt > 0)
// - posts with metadata changes (acknowledgements/priority)
hasMetadataChanges := p.Metadata != nil && (p.Metadata.Acknowledgements != nil || p.Metadata.Priority != nil)
if p.EditAt > 0 && p.EditAt < sd.scr.LastPostUpdateAt && p.DeleteAt == 0 && !hasMetadataChanges {
continue
}
// Don't send a deleted post if it is just the original copy from an edit.
if p.DeleteAt > 0 && p.OriginalId != "" {
continue
}
// don't sync a post back to the remote cluster it came from.
if p.GetRemoteID() == sd.rc.RemoteId {
continue
}
// parse out all permalinks in the message.
p.Message = scs.processPermalinkToRemote(p)
filtered = append(filtered, p)
}
sd.posts = filtered
}
// sendSyncData sends all the collected users, posts, reactions, images, and attachments to the
// remote cluster.
// The order of items sent is important: users -> memberships -> attachments -> posts -> reactions -> profile images
func (scs *Service) sendSyncData(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncSendDuration(sd.rc.RemoteId, time.Since(start).Seconds())
}
}()
merr := merror.New()
sanitizeSyncData(sd)
// send users
if len(sd.users) != 0 {
if err := scs.sendUserSyncData(sd); err != nil {
merr.Append(fmt.Errorf("cannot send user sync data: %w", err))
}
}
// send membership changes (after users so the remote has profiles for added members)
if len(sd.membershipChanges) != 0 {
if err := scs.sendMembershipSyncData(sd); err != nil {
merr.Append(fmt.Errorf("cannot send membership sync data: %w", err))
}
}
// send attachments
if len(sd.attachments) != 0 {
scs.sendAttachmentSyncData(sd)
}
// send posts
if len(sd.posts) != 0 {
if err := scs.sendPostSyncData(sd); err != nil {
merr.Append(fmt.Errorf("cannot send post sync data: %w", err))
}
} else if sd.isCursorChanged() {
scs.updateCursorForRemote(sd.scr.Id, sd.rc, sd.resultNextCursor)
}
// send acknowledgements
if len(sd.acknowledgements) != 0 {
if err := scs.sendAcknowledgementSyncData(sd); err != nil {
merr.Append(fmt.Errorf("cannot send acknowledgement sync data: %w", err))
}
}
// send reactions
if len(sd.reactions) != 0 {
if err := scs.sendReactionSyncData(sd); err != nil {
merr.Append(fmt.Errorf("cannot send reaction sync data: %w", err))
}
}
// send statuses
if len(sd.statuses) != 0 {
if err := scs.sendStatusSyncData(sd); err != nil {
merr.Append(fmt.Errorf("cannot send status sync data: %w", err))
}
}
// send user profile images
if len(sd.profileImages) != 0 {
scs.sendProfileImageSyncData(sd)
}
return merr.ErrorOrNil()
}
// sendUserSyncData sends the collected user updates to the remote cluster.
func (scs *Service) sendUserSyncData(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncSendStepDuration(sd.rc.RemoteId, "Users", time.Since(start).Seconds())
}
}()
msg := model.NewSyncMsg(sd.task.channelID)
msg.Users = sd.users
err := scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
// Only update cursor on successful sync
if errResp == nil && sd.GlobalUserSyncLastTimestamp > 0 {
scs.updateGlobalSyncCursor(sd.rc, sd.GlobalUserSyncLastTimestamp)
}
for _, userID := range syncResp.UsersSyncd {
if err := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(userID, sd.task.channelID, sd.rc.RemoteId); err != nil {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Cannot update shared channel user LastSyncAt",
mlog.String("user_id", userID),
mlog.String("channel_id", sd.task.channelID),
mlog.String("remote_id", sd.rc.RemoteId),
mlog.Err(err),
)
}
}
if len(syncResp.UserErrors) != 0 {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Response indicates error for user(s) sync",
mlog.String("channel_id", sd.task.channelID),
mlog.String("remote_id", sd.rc.RemoteId),
mlog.Array("users", syncResp.UserErrors),
)
}
})
return err
}
// sendAttachmentSyncData sends the collected post updates to the remote cluster.
func (scs *Service) sendAttachmentSyncData(sd *syncData) {
for _, a := range sd.attachments {
if err := scs.sendAttachmentForRemote(a.fi, a.post, sd.rc); err != nil {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Cannot sync post attachment",
mlog.String("post_id", a.post.Id),
mlog.String("channel_id", sd.task.channelID),
mlog.String("remote_id", sd.rc.RemoteId),
mlog.Err(err),
)
}
// updating SharedChannelAttachments with LastSyncAt is already done.
}
}
// sendPostSyncData sends the collected post updates to the remote cluster.
func (scs *Service) sendPostSyncData(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncSendStepDuration(sd.rc.RemoteId, "Posts", time.Since(start).Seconds())
}
}()
msg := model.NewSyncMsg(sd.task.channelID)
msg.Posts = sd.posts
msg.MentionTransforms = sd.mentionTransforms
return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
if len(syncResp.PostErrors) != 0 {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Response indicates error for post(s) sync",
mlog.String("channel_id", sd.task.channelID),
mlog.String("remote_id", sd.rc.RemoteId),
mlog.Array("posts", syncResp.PostErrors),
)
for _, postID := range syncResp.PostErrors {
scs.handlePostError(postID, sd.task, sd.rc)
}
}
scs.updateCursorForRemote(sd.scr.Id, sd.rc, sd.resultNextCursor)
})
}
// sendReactionSyncData sends the collected reaction updates to the remote cluster.
func (scs *Service) sendReactionSyncData(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncSendStepDuration(sd.rc.RemoteId, "Reactions", time.Since(start).Seconds())
}
}()
msg := model.NewSyncMsg(sd.task.channelID)
msg.Reactions = sd.reactions
return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
if len(syncResp.ReactionErrors) != 0 {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Response indicates error for reactions(s) sync",
mlog.String("channel_id", sd.task.channelID),
mlog.String("remote_id", sd.rc.RemoteId),
mlog.Array("reaction_posts", syncResp.ReactionErrors),
)
}
})
}
// sendAcknowledgementSyncData sends the collected acknowledgement updates to the remote cluster.
func (scs *Service) sendAcknowledgementSyncData(sd *syncData) error {
start := time.Now()
defer func() {
if metrics := scs.server.GetMetrics(); metrics != nil {
metrics.ObserveSharedChannelsSyncSendStepDuration(sd.rc.RemoteId, "Acknowledgements", time.Since(start).Seconds())
}
}()
msg := model.NewSyncMsg(sd.task.channelID)
msg.Acknowledgements = sd.acknowledgements
return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
if len(syncResp.AcknowledgementErrors) != 0 {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Response indicates error for acknowledgement(s) sync",
mlog.String("channel_id", sd.task.channelID),
mlog.String("remote_id", sd.rc.RemoteId),
mlog.Array("acknowledgement_posts", syncResp.AcknowledgementErrors),
)
}
})
}
// sendStatusSyncData sends the collected status updates to the remote cluster.
func (scs *Service) sendStatusSyncData(sd *syncData) error {
msg := model.NewSyncMsg(sd.task.channelID)
msg.Statuses = sd.statuses
return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
if len(syncResp.StatusErrors) != 0 {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Response indicates error from status(es) sync",
mlog.String("remote_id", sd.rc.RemoteId),
mlog.Array("user_ids", syncResp.StatusErrors),
)
for _, userID := range syncResp.StatusErrors {
scs.handleStatusError(userID, sd.task, sd.rc)
}
}
})
}
// sendProfileImageSyncData sends the collected user profile image updates to the remote cluster.
func (scs *Service) sendProfileImageSyncData(sd *syncData) {
for _, user := range sd.profileImages {
scs.syncProfileImage(user, sd.task.channelID, sd.rc)
}
}
// shouldUserSyncGlobal determines if a user needs to be synchronized globally.
// Compares user's update timestamp with the remote cluster's LastGlobalUserSyncAt.
func (scs *Service) shouldUserSyncGlobal(user *model.User, rc *model.RemoteCluster) (bool, error) {
// Don't sync users back to the remote cluster they originated from
if user.IsRemote() && user.GetRemoteID() == rc.RemoteId {
return false, nil
}
// Calculate latest update time for this user (profile or picture)
latestUserUpdateTime := max(user.LastPictureUpdate, user.UpdateAt)
// For initial sync (LastGlobalUserSyncAt=0), sync all users
// For incremental sync, only sync users updated after the last sync
if rc.LastGlobalUserSyncAt == 0 {
return true, nil
}
return latestUserUpdateTime > rc.LastGlobalUserSyncAt, nil
}
// syncAllUsers synchronizes all local users to a remote cluster.
// This is called when a connection with a remote cluster is established or when handling a global user sync task.
// Uses cursor-based approach with LastGlobalUserSyncAt to resume after interruptions.
func (scs *Service) syncAllUsers(rc *model.RemoteCluster) error {
// Check if feature is enabled
if !scs.server.Config().FeatureFlags.EnableSyncAllUsersForRemoteCluster {
return nil
}
if !rc.IsOnline() {
return fmt.Errorf("remote cluster %s is not online", rc.RemoteId)
}
// Start metrics tracking
metrics := scs.server.GetMetrics()
start := time.Now()
defer func() {
if metrics != nil {
metrics.IncrementSharedChannelsSyncCounter(rc.RemoteId)
metrics.ObserveSharedChannelsSyncCollectionDuration(rc.RemoteId, time.Since(start).Seconds())
}
}()
batchSize := scs.getGlobalUserSyncBatchSize()
// Create sync data with collected users
sd := &syncData{
task: syncTask{remoteID: rc.RemoteId},
rc: rc,
scr: &model.SharedChannelRemote{RemoteId: rc.RemoteId},
users: make(map[string]*model.User),
}
// Collect users to sync
users, latestTimestamp, _, hasMore, err := scs.collectUsersForGlobalSync(rc, batchSize)
if err != nil {
return err
}
// Exit early if no users to sync
if len(users) == 0 {
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "No users to sync for remote cluster",
mlog.String("remote_id", rc.RemoteId))
return nil
}
// Add users to sync data
sd.users = users
sd.GlobalUserSyncLastTimestamp = latestTimestamp
// Send the collected users to remote
if err := scs.sendUserSyncData(sd); err != nil {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceError, "Error sending user batch during sync",
mlog.String("remote_id", rc.RemoteId),
mlog.Err(err),
)
return fmt.Errorf("error sending user batch during sync: %w", err)
}
// Schedule next batch if needed
if hasMore {
scs.scheduleNextUserSyncBatch(rc, latestTimestamp, batchSize, len(users))
}
return nil
}
// getGlobalUserSyncBatchSize returns the configured batch size for user syncing
func (scs *Service) getGlobalUserSyncBatchSize() int {
batchSize := MaxUsersPerSync
if scs.server.Config().ConnectedWorkspacesSettings.GlobalUserSyncBatchSize != nil {
configValue := *scs.server.Config().ConnectedWorkspacesSettings.GlobalUserSyncBatchSize
if configValue > 0 && configValue <= 200 {
batchSize = configValue
}
}
return batchSize
}
// collectUsersForGlobalSync fetches users that need to be synced to the remote
func (scs *Service) collectUsersForGlobalSync(rc *model.RemoteCluster, batchSize int) (map[string]*model.User, int64, int, bool, error) {
options := &model.UserGetOptions{
Page: 0,
PerPage: 100, // Database fetch batch size
Active: true,
Sort: "update_at_asc", // Order by UpdateAt ASC to ensure cursor consistency
}
// Only use UpdatedAfter for incremental syncs, not the initial sync
// This ensures users with UpdateAt=0 are included in the first sync
if rc.LastGlobalUserSyncAt > 0 {
options.UpdatedAfter = rc.LastGlobalUserSyncAt
}
users := make(map[string]*model.User)
latestTimestamp := rc.LastGlobalUserSyncAt
totalCount := 0
// Page through database results
for {
batch, err := scs.server.GetStore().User().GetAllProfiles(options)
if err != nil {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceError, "Error fetching users for global sync",
mlog.String("remote_id", rc.RemoteId),
mlog.Err(err),
)
return nil, 0, 0, false, err
}
if len(batch) == 0 {
break // No more users to process
}
totalCount += len(batch)
// Process each user in this database page
for _, user := range batch {
// Stop if we've reached batch limit
if len(users) >= batchSize {
return users, latestTimestamp, totalCount, true, nil
}
// Skip users from remotes
if user.IsRemote() {
continue
}
// Check if user needs syncing
needsSync, _ := scs.shouldUserSyncGlobal(user, rc)
if !needsSync {
continue
}
// Add user and update cursor timestamp
users[user.Id] = user
userUpdateTime := max(user.UpdateAt, user.LastPictureUpdate)
if userUpdateTime > latestTimestamp {
latestTimestamp = userUpdateTime
}
}
// Check if we've reached the end of results
if len(batch) < options.PerPage {
break
}
// Move to next page
options.Page++
}
return users, latestTimestamp, totalCount, false, nil
}
// updateGlobalSyncCursor updates the LastGlobalUserSyncAt value for the remote cluster
func (scs *Service) updateGlobalSyncCursor(rc *model.RemoteCluster, newTimestamp int64) {
if err := scs.server.GetStore().RemoteCluster().UpdateLastGlobalUserSyncAt(rc.RemoteId, newTimestamp); err == nil {
rc.LastGlobalUserSyncAt = newTimestamp
} else {
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceError, "Failed to update global user sync cursor",
mlog.String("remote_id", rc.RemoteId),
mlog.Err(err),
)
}
}
// scheduleNextUserSyncBatch creates a new task for the next batch of user sync
func (scs *Service) scheduleNextUserSyncBatch(rc *model.RemoteCluster, timestamp int64, batchSize, processedCount int) {
// Use timestamp as userID to make each batch task unique
// This prevents task ID collisions between different batches for the same remote
timestampStr := fmt.Sprintf("%d", timestamp)
task := newSyncTask("", timestampStr, rc.RemoteId, nil, nil)
task.schedule = time.Now().Add(NotifyMinimumDelay)
scs.addTask(task)
}
// sendSyncMsgToRemote synchronously sends the sync message to the remote cluster (or plugin).
func (scs *Service) sendSyncMsgToRemote(msg *model.SyncMsg, rc *model.RemoteCluster, f sendSyncMsgResultFunc) error {
rcs := scs.server.GetRemoteClusterService()
if rcs == nil {
return fmt.Errorf("cannot update remote cluster %s for channel id %s; Remote Cluster Service not enabled", rc.Name, msg.ChannelId)
}
if rc.IsPlugin() {
return scs.sendSyncMsgToPlugin(msg, rc, f)
}
b, err := json.Marshal(msg)
if err != nil {
return err
}
// Use appropriate topic based on message type
topic := TopicSync
if msg.ChannelId == "" && len(msg.Users) > 0 &&
len(msg.Posts) == 0 && len(msg.Reactions) == 0 &&
len(msg.Statuses) == 0 {
topic = TopicGlobalUserSync
}
rcMsg := model.NewRemoteClusterMsg(topic, b)
ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
defer cancel()
var wg sync.WaitGroup
wg.Add(1)
err = rcs.SendMsg(ctx, rcMsg, rc, func(rcMsg model.RemoteClusterMsg, rc *model.RemoteCluster, rcResp *remotecluster.Response, errResp error) {
defer wg.Done()
// Check for ErrChannelNotShared in the application response
if rcResp != nil && !rcResp.IsSuccess() && strings.Contains(rcResp.Err, ErrChannelNotShared.Error()) {
scs.handleChannelNotSharedError(msg, rc)
return
}
// Any failure to confirm delivery means the content was not applied: a transport
// error (remote unreachable) or a non-success response envelope reported by the
// remote. The send is fire-and-forget, so instead of surfacing an error to the
// caller we signal the remote cluster service: on its next successful ping it
// fires a connection-state-change event that triggers ForceSyncForRemote,
// re-syncing this channel once the remote is reachable again. The cursor is only
// advanced on success (via the result callback below), so nothing is lost in the
// meantime. Checking IsSuccess() here is essential: without it a failed envelope
// carrying a JSON body would be unmarshalled and passed to f, advancing the cursor
// as if the sync had succeeded.
if errResp != nil || (rcResp != nil && !rcResp.IsSuccess()) {
scs.notifyRemoteSyncFailed(rc)
return
}
var syncResp model.SyncResponse
if rcResp != nil && len(rcResp.Payload) > 0 {
if err2 := json.Unmarshal(rcResp.Payload, &syncResp); err2 != nil {
// A 200 response whose body we cannot parse: we can't confirm the sync
// landed, so leave the cursor un-advanced and trigger the same ping-driven
// resync as a delivery failure.
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Invalid sync msg response from remote cluster",
mlog.String("remote", rc.Name),
mlog.String("channel_id", msg.ChannelId),
mlog.Err(err2),
)
scs.notifyRemoteSyncFailed(rc)
return
}
if f != nil {
f(syncResp, errResp)
}
} else {
// A 200 with no payload. The receive side always sets a SyncResponse payload on
// success (onReceiveSyncMessage), so an empty payload means the sync was not
// confirmed. Treat it like an unparseable response: don't advance the cursor and
// trigger a ping-driven resync.
scs.server.Log().LogM(mlog.MlvlSharedChannelServiceWarn, "Empty or nil response payload from remote cluster",
mlog.String("remote", rc.Name),
mlog.String("channel_id", msg.ChannelId),
)
scs.notifyRemoteSyncFailed(rc)
}
})
if err != nil {
// The enqueue failed (e.g. BufferFullError), so the callback never runs and
// wg.Done() is never called; returning here avoids blocking on wg.Wait() forever.
return err
}
wg.Wait()
return err
}
// notifyRemoteSyncFailed signals the remote cluster service that a sync to rc failed to
// be delivered. On the remote's next successful ping this drives a connection-state-change
// event and a ForceSyncForRemote, recovering content that could not be sent while the
// remote was briefly unreachable (including short outages where IsOnline() never flipped).
func (scs *Service) notifyRemoteSyncFailed(rc *model.RemoteCluster) {
if rcs := scs.server.GetRemoteClusterService(); rcs != nil {
rcs.NotifySyncFailed(rc.RemoteId)
}
}
// sendSyncMsgToRemote synchronously sends the sync message to a plugin.
func (scs *Service) sendSyncMsgToPlugin(msg *model.SyncMsg, rc *model.RemoteCluster, f sendSyncMsgResultFunc) error {
syncResp, errResp := scs.app.OnSharedChannelsSyncMsg(msg, rc)
if f != nil {
f(syncResp, errResp)
}
return errResp
}
func sanitizeSyncData(sd *syncData) {
for id, user := range sd.users {
sd.users[id] = sanitizeUserForSync(user)
}
for id, user := range sd.profileImages {
sd.profileImages[id] = sanitizeUserForSync(user)
}
}
// handleChannelNotSharedError processes the case when a remote indicates a channel
// is no longer shared. It removes the remote from the shared channel locally and,
// if it was the last remote, completely unshares the channel.
func (scs *Service) handleChannelNotSharedError(msg *model.SyncMsg, rc *model.RemoteCluster) {
logger := scs.server.Log()
logger.Log(mlog.LvlSharedChannelServiceDebug, "Remote indicated channel is no longer shared; unsharing locally",
mlog.String("remote", rc.Name),
mlog.String("channel_id", msg.ChannelId),
)
if err := scs.UninviteRemoteFromChannel(msg.ChannelId, rc.RemoteId); err != nil {
logger.LogM(mlog.MlvlSharedChannelServiceError, "Failed to uninvite remote from shared channel", mlog.Err(err))
return
}
}