team/server/enterprise/elasticsearch/common/indexing_job_test.go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.enterprise for license information.

package common

import (
	"fmt"
	"io"
	"maps"
	"strconv"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"

	"github.com/mattermost/mattermost/server/public/model"
	"github.com/mattermost/mattermost/server/public/shared/mlog"
	"github.com/mattermost/mattermost/server/v8/channels/jobs"
	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)

func TestBulkIndexChannelsWithDeletedChannels(t *testing.T) {
	// Create test channels - one active, one deleted
	activeChannel := &model.Channel{
		Id:       "ch1",
		Type:     model.ChannelTypeOpen,
		DeleteAt: 0,
	}
	deletedChannel := &model.Channel{
		Id:       "ch2",
		Type:     model.ChannelTypeOpen,
		DeleteAt: 123456,
	}
	channels := []*model.Channel{activeChannel, deletedChannel}

	// Mock store
	mockStore := &mocks.Store{}
	mockChannelStore := &mocks.ChannelStore{}
	mockStore.On("Channel").Return(mockChannelStore)
	defer mockStore.AssertExpectations(t)

	// Since these are open channels, GetAllChannelMemberIdsByChannelId won't be called
	// But GetTeamMembersForChannel will be called for both channels
	mockChannelStore.On("GetTeamMembersForChannel", mock.AnythingOfType("*request.Context"), "ch1").Return([]string{"team1"}, nil)
	mockChannelStore.On("GetTeamMembersForChannel", mock.AnythingOfType("*request.Context"), "ch2").Return([]string{"team1"}, nil)

	// Track which channels were actually indexed
	indexedChannels := make(map[string]bool)

	// Mock bulk processor function
	addItemToBulkProcessorFn := func(_, op, id string, _ io.ReadSeeker) error {
		assert.Equal(t, indexOp, op) // Should always be index, not delete
		indexedChannels[id] = true
		return nil
	}

	config := &model.Config{}
	config.ElasticsearchSettings.IndexPrefix = new("test_")

	// Call the function
	lastChannel, appErr := BulkIndexChannels(config, mockStore, mlog.CreateConsoleTestLogger(t), addItemToBulkProcessorFn, channels, IndexingProgress{})

	// Verify results
	require.Nil(t, appErr)
	assert.Equal(t, deletedChannel, lastChannel)
	assert.True(t, indexedChannels["ch1"], "Active channel should be indexed")
	assert.True(t, indexedChannels["ch2"], "Deleted channel should also be indexed")
}

func TestCurrentProgressCapsAt100(t *testing.T) {
	tests := []struct {
		name     string
		progress IndexingProgress
		expected int64
	}{
		{
			name: "normal progress",
			progress: IndexingProgress{
				DonePostsCount:     50,
				TotalPostsCount:    100,
				DoneChannelsCount:  0,
				TotalChannelsCount: 100,
				DoneUsersCount:     0,
				TotalUsersCount:    100,
				DoneFilesCount:     0,
				TotalFilesCount:    100,
			},
			expected: 12,
		},
		{
			name: "exactly 100%",
			progress: IndexingProgress{
				DonePostsCount:     100,
				TotalPostsCount:    100,
				DoneChannelsCount:  50,
				TotalChannelsCount: 50,
				DoneUsersCount:     10,
				TotalUsersCount:    10,
				DoneFilesCount:     5,
				TotalFilesCount:    5,
			},
			expected: 100,
		},
		{
			name:     "all totals zero returns 100",
			progress: IndexingProgress{},
			expected: 100,
		},
		{
			name: "done exceeds total, caps at 100",
			progress: IndexingProgress{
				DonePostsCount:     20000000,
				TotalPostsCount:    10000000,
				DoneChannelsCount:  0,
				TotalChannelsCount: 100000,
				DoneUsersCount:     0,
				TotalUsersCount:    10000,
				DoneFilesCount:     0,
				TotalFilesCount:    100000,
			},
			expected: 100,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert.Equal(t, tt.expected, tt.progress.CurrentProgress())
		})
	}
}

func TestEntityCountFallback(t *testing.T) {
	tests := []struct {
		name      string
		jobData   model.StringMap
		dataKey   string
		estimate  int64
		doneCount int64
		expected  int64
	}{
		{
			name:      "no stored value, uses estimate",
			jobData:   model.StringMap{},
			dataKey:   "total_posts_count",
			estimate:  10000000,
			doneCount: 0,
			expected:  10000000,
		},
		{
			name:      "stored value preferred over estimate",
			jobData:   model.StringMap{"total_posts_count": "5000000"},
			dataKey:   "total_posts_count",
			estimate:  10000000,
			doneCount: 0,
			expected:  5000000,
		},
		{
			name:      "doneCount exceeds stored value",
			jobData:   model.StringMap{"total_posts_count": "5000000"},
			dataKey:   "total_posts_count",
			estimate:  10000000,
			doneCount: 7000000,
			expected:  7000000,
		},
		{
			name:      "doneCount exceeds estimate when no stored value",
			jobData:   model.StringMap{},
			dataKey:   "total_posts_count",
			estimate:  10000000,
			doneCount: 15000000,
			expected:  15000000,
		},
		{
			name:      "invalid stored value falls back to estimate",
			jobData:   model.StringMap{"total_posts_count": "not-a-number"},
			dataKey:   "total_posts_count",
			estimate:  10000000,
			doneCount: 0,
			expected:  10000000,
		},
		{
			name:      "zero stored value falls back to estimate",
			jobData:   model.StringMap{"total_posts_count": "0"},
			dataKey:   "total_posts_count",
			estimate:  10000000,
			doneCount: 0,
			expected:  10000000,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			job := &model.Job{Data: tt.jobData}
			result := entityCountFallback(job, tt.dataKey, tt.estimate, tt.doneCount)
			assert.Equal(t, tt.expected, result)
		})
	}
}

func TestSetEntityCount(t *testing.T) {
	type entityCountMocks struct {
		store    *mocks.Store
		post     *mocks.PostStore
		channel  *mocks.ChannelStore
		user     *mocks.UserStore
		fileInfo *mocks.FileInfoStore
	}

	setupEntityCountMocks := func() entityCountMocks {
		m := entityCountMocks{
			store:    &mocks.Store{},
			post:     &mocks.PostStore{},
			channel:  &mocks.ChannelStore{},
			user:     &mocks.UserStore{},
			fileInfo: &mocks.FileInfoStore{},
		}
		m.store.On("Post").Return(m.post)
		m.store.On("Channel").Return(m.channel)
		m.store.On("User").Return(m.user)
		m.store.On("FileInfo").Return(m.fileInfo)
		return m
	}

	allEntitiesEnabled := model.StringMap{
		"index_posts":    "true",
		"index_channels": "true",
		"index_users":    "true",
		"index_files":    "true",
	}

	t.Run("stores counts on success", func(t *testing.T) {
		m := setupEntityCountMocks()
		m.post.On("AnalyticsPostCount", mock.Anything).Return(int64(500), nil)
		m.channel.On("AnalyticsTypeCount", "", model.ChannelType("")).Return(int64(200), nil)
		m.user.On("Count", mock.Anything).Return(int64(50), nil)
		m.fileInfo.On("CountAll").Return(int64(300), nil)

		job := &model.Job{Data: maps.Clone(allEntitiesEnabled)}
		progress := setEntityCount(mlog.CreateConsoleTestLogger(t), &jobs.JobServer{Store: m.store}, IndexingProgress{}, job)

		assert.Equal(t, int64(500), progress.TotalPostsCount)
		assert.Equal(t, int64(200), progress.TotalChannelsCount)
		assert.Equal(t, int64(50), progress.TotalUsersCount)
		assert.Equal(t, int64(300), progress.TotalFilesCount)

		assert.Equal(t, "500", job.Data["total_posts_count"])
		assert.Equal(t, "200", job.Data["total_channels_count"])
		assert.Equal(t, "50", job.Data["total_users_count"])
		assert.Equal(t, "300", job.Data["total_files_count"])
	})

	t.Run("falls back to job data on query failure", func(t *testing.T) {
		m := setupEntityCountMocks()
		m.post.On("AnalyticsPostCount", mock.Anything).Return(int64(0), fmt.Errorf("timeout"))
		m.channel.On("AnalyticsTypeCount", "", model.ChannelType("")).Return(int64(0), fmt.Errorf("timeout"))
		m.user.On("Count", mock.Anything).Return(int64(0), fmt.Errorf("timeout"))
		m.fileInfo.On("CountAll").Return(int64(0), fmt.Errorf("timeout"))

		jobData := maps.Clone(allEntitiesEnabled)
		jobData["total_posts_count"] = "8000000"
		jobData["total_channels_count"] = "50000"
		jobData["total_users_count"] = "5000"
		jobData["total_files_count"] = "75000"
		job := &model.Job{Data: jobData}

		progress := setEntityCount(mlog.CreateConsoleTestLogger(t), &jobs.JobServer{Store: m.store}, IndexingProgress{}, job)

		assert.Equal(t, int64(8000000), progress.TotalPostsCount)
		assert.Equal(t, int64(50000), progress.TotalChannelsCount)
		assert.Equal(t, int64(5000), progress.TotalUsersCount)
		assert.Equal(t, int64(75000), progress.TotalFilesCount)
	})

	t.Run("uses max of fallback and done count", func(t *testing.T) {
		m := setupEntityCountMocks()
		m.post.On("AnalyticsPostCount", mock.Anything).Return(int64(0), fmt.Errorf("timeout"))

		job := &model.Job{
			Data: model.StringMap{
				"index_posts":       "true",
				"index_channels":    "false",
				"index_users":       "false",
				"index_files":       "false",
				"total_posts_count": strconv.FormatInt(estimatedPostCount, 10),
			},
		}
		inputProgress := IndexingProgress{
			DonePostsCount: estimatedPostCount + 5000000,
		}

		progress := setEntityCount(mlog.CreateConsoleTestLogger(t), &jobs.JobServer{Store: m.store}, inputProgress, job)

		assert.Equal(t, int64(estimatedPostCount+5000000), progress.TotalPostsCount)
	})
}