package br import ( "bytes" "context" "encoding/gob" "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" clusterngv1beta "github.com/tidbcloud/cluster-service-ng/api/cluster-service-ng/v1beta" "go.uber.org/mock/gomock" mockregionalserver "github.com/tidbcloud/tidb-management-service/nextgen-global-service/domain/adapter/regionalserver/mock_regionalserver" "github.com/tidbcloud/tidb-management-service/nextgen-global-service/domain/entity" brrepo "github.com/tidbcloud/tidb-management-service/nextgen-global-service/domain/repository/br" mockbrrepo "github.com/tidbcloud/tidb-management-service/nextgen-global-service/domain/repository/br/mock_br" mockclusterrepo "github.com/tidbcloud/tidb-management-service/nextgen-global-service/domain/repository/cluster/mock_cluster" mockregionrepo "github.com/tidbcloud/tidb-management-service/nextgen-global-service/domain/repository/region/mock_region" "github.com/tidbcloud/tidb-management-service/nextgen-global-service/infra/repoimpl/schema" "github.com/tidbcloud/tidb-management-service/nextgen-global-service/pkg/config" "github.com/tidbcloud/tidb-management-service/nextgen-global-service/pkg/externalmocks" "github.com/tidbcloud/tidb-management-service/utilities/bizerr" "github.com/tidbcloud/tidb-management-service/utilities/pagination" ) func TestBackupSyncSlots_rebuildWithoutLock(t *testing.T) { t.Run("one per-day, one per-hour, one per-minute", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() bs1 := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 1, SharedPoolId: 1, Schedule: schema.AutoSchedule{ Type: "per-day", At: 1 * 60, }, }} bs2 := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 2, SharedPoolId: 1, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }} bs3 := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 3, SharedPoolId: 2, Schedule: schema.AutoSchedule{ Type: "per-minute", At: 0, }, }} backupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), &brrepo.QueryBackupSettingFilter{ States: []schema.BackupSettingState{schema.BackupSettingStateCreated}, }, nil).Return([]*entity.BackupSetting{bs1, bs2, bs3}, nil) slots := newBackupSyncSlots(nil, backupSettingRepo, nil, nil, nil, nil, nil, nil) err := slots.rebuildWithoutLock(t.Context()) require.NoError(t, err) assert.Equal(t, map[uint64]*entity.BackupSetting{1: bs1, 2: bs2, 3: bs3}, slots.backupSettings) // assert the cluster slots assert.Equal(t, []int{60}, slots.clusterSlots[1]) cluster2Slots := []int{ 0, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 660, 720, 780, 840, 900, 960, 1020, 1080, 1140, 1200, 1260, 1320, 1380, } assert.Equal(t, cluster2Slots, slots.clusterSlots[2]) cluster3Slots := []int{} for i := range 24 * 60 { cluster3Slots = append(cluster3Slots, i) } assert.Equal(t, cluster3Slots, slots.clusterSlots[3]) // assert the slots // overlap slot with 3 clusters assert.Len(t, slots.slots[60], 2) // two shared pools assert.Len(t, slots.slots[60][1], 2) // two clusters assert.Equal(t, map[uint64]bool{1: true, 2: true}, slots.slots[60][1]) assert.Len(t, slots.slots[60][2], 1) // one cluster assert.Equal(t, map[uint64]bool{3: true}, slots.slots[60][2]) // overlap slot with 2 clusters for _, slot := range cluster2Slots { if slot == 60 { continue } assert.Len(t, slots.slots[slot], 2) assert.Len(t, slots.slots[slot][1], 1) assert.Equal(t, map[uint64]bool{2: true}, slots.slots[slot][1]) assert.Len(t, slots.slots[slot][2], 1) assert.Equal(t, map[uint64]bool{3: true}, slots.slots[slot][2]) } // for non overlap slot for _, slot := range cluster3Slots { if slot%60 == 0 { continue } assert.Len(t, slots.slots[slot], 1) assert.Len(t, slots.slots[slot][2], 1) assert.Equal(t, map[uint64]bool{3: true}, slots.slots[slot][2]) } }) t.Run("no backup setting", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() backupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), &brrepo.QueryBackupSettingFilter{ States: []schema.BackupSettingState{schema.BackupSettingStateCreated}, }, nil).Return([]*entity.BackupSetting{}, nil) slots := newBackupSyncSlots(nil, backupSettingRepo, nil, nil, nil, nil, nil, nil) err := slots.rebuildWithoutLock(t.Context()) require.NoError(t, err) assert.Equal(t, map[uint64]*entity.BackupSetting{}, slots.backupSettings) assert.Empty(t, slots.clusterSlots) for _, slot := range slots.slots { assert.Empty(t, slot) } }) t.Run("query backup setting failed", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() backupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), &brrepo.QueryBackupSettingFilter{ States: []schema.BackupSettingState{schema.BackupSettingStateCreated}, }, nil).Return(nil, bizerr.New(bizerr.InternalErrCodeCustomEnd, "failed to query backup setting")) slots := newBackupSyncSlots(nil, backupSettingRepo, nil, nil, nil, nil, nil, nil) err := slots.rebuildWithoutLock(t.Context()) require.Error(t, err) }) } func TestCalculateSlotsIndexesFlexibleRetentionUsesOnlyDailyBackupTime(t *testing.T) { flexible := &entity.BackupSetting{BackupSetting: &schema.BackupSetting{ Mode: schema.BackupModeFlexibleRetention, Schedule: schema.AutoSchedule{ Type: "per-day", At: 5*60 + 3, DailyBackupMinuteOfDay: 5*60 + 3, }, }} standard := &entity.BackupSetting{BackupSetting: &schema.BackupSetting{ Mode: schema.BackupModeStandardBundle, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 3, }, }} assert.Equal(t, []int{5*60 + 3}, calculateSlotsIndexes(flexible)) assert.Len(t, calculateSlotsIndexes(standard), 24) assert.Equal(t, 3, calculateSlotsIndexes(standard)[0]) assert.Equal(t, 23*60+3, calculateSlotsIndexes(standard)[23]) } func TestGetAutoBackupType(t *testing.T) { tests := []struct { name string slotTime time.Time mode schema.BackupMode schedule schema.AutoSchedule expectedType schema.AutoBackupType }{ { name: "flexible backup is daily", slotTime: time.Date(2026, 8, 21, 3, 0, 0, 0, time.UTC), mode: schema.BackupModeFlexibleRetention, schedule: schema.AutoSchedule{Type: "per-hour", DailyBackupMinuteOfDay: 5 * 60}, expectedType: schema.AutoBackupTypeDaily, }, { name: "per-day backup is daily", slotTime: time.Date(2026, 8, 21, 5, 0, 0, 0, time.UTC), mode: schema.BackupModeStandardBundle, schedule: schema.AutoSchedule{Type: "per-day"}, expectedType: schema.AutoBackupTypeDaily, }, { name: "daily slot in hourly schedule is daily", slotTime: time.Date(2026, 8, 21, 5, 3, 0, 0, time.UTC), mode: schema.BackupModeStandardBundle, schedule: schema.AutoSchedule{Type: "per-hour", DailyBackupMinuteOfDay: 5*60 + 3}, expectedType: schema.AutoBackupTypeDaily, }, { name: "non-daily slot in hourly schedule is hourly", slotTime: time.Date(2026, 8, 21, 6, 3, 0, 0, time.UTC), mode: schema.BackupModeStandardBundle, schedule: schema.AutoSchedule{Type: "per-hour", DailyBackupMinuteOfDay: 5*60 + 3}, expectedType: schema.AutoBackupTypeHourly, }, { name: "unsupported schedule is unknown", slotTime: time.Date(2026, 8, 21, 5, 3, 0, 0, time.UTC), mode: schema.BackupModeStandardBundle, schedule: schema.AutoSchedule{Type: "per-minute", DailyBackupMinuteOfDay: 5*60 + 3}, expectedType: schema.AutoBackupTypeUnknown, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { setting := &entity.BackupSetting{BackupSetting: &schema.BackupSetting{ Mode: tt.mode, Schedule: tt.schedule, }} assert.Equal(t, tt.expectedType, getAutoBackupType(tt.slotTime, setting)) }) } } func TestParseBackupTime(t *testing.T) { expected := time.Date(2026, 2, 10, 7, 57, 15, 0, time.UTC) tests := []struct { name string value string }{ { name: "with milliseconds", value: "2026-02-10T07:57:15.000Z", }, { name: "without milliseconds", value: "2026-02-10T07:57:15Z", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { got, err := parseBackupTime(tc.value) require.NoError(t, err) assert.True(t, got.Equal(expected)) }) } } func TestBackupSyncSlots_calculateToBeSyncedMinutes(t *testing.T) { traceBackHours := 12 cfg := &config.SyncBackupsConfig{ TraceBackHours: traceBackHours, } t.Run("currentSyncedCompletedSlotIndex is nil - process restart scenario, no cross day", func(t *testing.T) { // Test when process restarts (currentSyncedCompletedSlotIndex is nil) // Should trace back TraceBackHours (12 hours) from target catchup time now := time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC) // 14:30 currentMinuteOfDay := 14*60 + 30 // 870 targetCatchupMinuteOfDay := currentMinuteOfDay - 5 // 865 expectedBackTraceMinutes := traceBackHours * 60 // 12 * 60 = 720 expectedStartSyncMinute := targetCatchupMinuteOfDay - expectedBackTraceMinutes // 865 - 720 = 145 result := calculateToBeSyncedMinutes(t.Context(), nil, now, cfg) // Should return minutes from startSyncMinute to targetCatchupMinuteOfDay expectedLength := targetCatchupMinuteOfDay - expectedStartSyncMinute + 1 // 865 - 145 + 1 = 721 assert.Len(t, result, expectedLength) // Check first and last elements assert.Equal(t, expectedStartSyncMinute, result[0]) assert.Equal(t, targetCatchupMinuteOfDay, result[len(result)-1]) // Check all elements are sequential for i := 1; i < len(result); i++ { assert.Equal(t, result[i-1]+1, result[i]) } }) t.Run("currentSyncedCompletedSlotIndex is nil - process restart scenario, cross day", func(t *testing.T) { now := time.Date(2025, 1, 15, 10, 30, 0, 0, time.UTC) // 10:30 currentMinuteOfDay := 10*60 + 30 // 630 targetCatchupMinuteOfDay := currentMinuteOfDay - 5 // 630 - 5 = 625 expectedBackTraceMinutes := traceBackHours * 60 // 12 * 60 = 720 result := calculateToBeSyncedMinutes(t.Context(), nil, now, cfg) assert.Len(t, result, expectedBackTraceMinutes+1) // Check first and last elements expectedStartSyncMinute := targetCatchupMinuteOfDay - expectedBackTraceMinutes assert.Equal(t, expectedStartSyncMinute, result[0]) assert.Equal(t, targetCatchupMinuteOfDay, result[len(result)-1]) // Check all elements are sequential for i := 1; i < len(result); i++ { assert.Equal(t, result[i-1]+1, result[i]) } }) t.Run("currentSyncedCompletedSlotIndex is not nil - normal sync scenario, no cross day", func(t *testing.T) { // Test normal sync scenario where we have a previous sync index now := time.Date(2025, 1, 15, 10, 15, 0, 0, time.UTC) // 10:15 currentMinuteOfDay := 10*60 + 15 // 615 targetCatchupMinuteOfDay := currentMinuteOfDay - 5 // 610 currentSyncedCompletedSlotIndex := 600 // Previous sync completed at minute 600 result := calculateToBeSyncedMinutes(t.Context(), ¤tSyncedCompletedSlotIndex, now, cfg) // Should return minutes from currentSyncedCompletedSlotIndex + 1 to targetCatchupMinuteOfDay expectedLength := targetCatchupMinuteOfDay - currentSyncedCompletedSlotIndex // 610 - 600 = 10 assert.Len(t, result, expectedLength) // Check first and last elements assert.Equal(t, currentSyncedCompletedSlotIndex+1, result[0]) assert.Equal(t, targetCatchupMinuteOfDay, result[len(result)-1]) // Check all elements are sequential for i := 1; i < len(result); i++ { assert.Equal(t, result[i-1]+1, result[i]) } }) t.Run("currentSyncedCompletedSlotIndex is not nil - normal sync scenario, cross day", func(t *testing.T) { // Test normal sync scenario where we have a previous sync index now := time.Date(2025, 1, 15, 0, 15, 0, 0, time.UTC) // 00:15 currentMinuteOfDay := 15 // 15 targetCatchupMinuteOfDay := currentMinuteOfDay - 5 // 10 currentSyncedCompletedSlotIndex := 1420 // Previous sync completed at minute 1420 result := calculateToBeSyncedMinutes(t.Context(), ¤tSyncedCompletedSlotIndex, now, cfg) // Should return minutes from currentSyncedCompletedSlotIndex + 1 to targetCatchupMinuteOfDay expectedLength := 30 // 10 + (1440 - 1420) = 30 assert.Len(t, result, expectedLength) // Check first and last elements assert.Equal(t, currentSyncedCompletedSlotIndex-MinuteCountOfADay+1, result[0]) assert.Equal(t, targetCatchupMinuteOfDay, result[len(result)-1]) // Check all elements are sequential for i := 1; i < len(result); i++ { assert.Equal(t, result[i-1]+1, result[i]) } }) } func TestBackupSyncSlots_update(t *testing.T) { cfg := &config.SyncBackupsConfig{ LoadScheduleChangeInterval: 5 * time.Minute, } bs1 := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 1, SharedPoolId: 1, Schedule: schema.AutoSchedule{ Type: "per-day", At: 1 * 60, }, }} bs2 := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 2, SharedPoolId: 1, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }} bs3 := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 3, SharedPoolId: 2, Schedule: schema.AutoSchedule{ Type: "per-minute", At: 0, }, }} bs4 := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 4, SharedPoolId: 2, Schedule: schema.AutoSchedule{ Type: "per-minute", At: 0, }, }} t.Run("no backup setting updated", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() backupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) // for building the slots backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), &brrepo.QueryBackupSettingFilter{ States: []schema.BackupSettingState{schema.BackupSettingStateCreated}, }, nil).Return([]*entity.BackupSetting{bs1, bs2, bs3}, nil) // for updating the slots - first call for recently schedule changed backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), gomock.Any(), nil).DoAndReturn( func(ctx context.Context, filter *brrepo.QueryBackupSettingFilter, page *pagination.Page) ([]*entity.BackupSetting, bizerr.BizError) { // Verify the filter for recently schedule changed backup settings assert.Equal(t, []schema.BackupSettingState{schema.BackupSettingStateCreated}, filter.States) assert.NotNil(t, filter.LastScheduleUpdatedAtGte) return []*entity.BackupSetting{}, nil }) // for updating the slots - second call for recently deleted backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), gomock.Any(), nil).DoAndReturn( func(ctx context.Context, filter *brrepo.QueryBackupSettingFilter, page *pagination.Page) ([]*entity.BackupSetting, bizerr.BizError) { // Verify the filter reconciles every deleted setting still present in this process's cache. assert.Equal(t, schema.BackupSettingStatesForDeletedCluster, filter.States) assert.ElementsMatch(t, []uint64{1, 2, 3}, filter.ClusterIds) assert.Nil(t, filter.DeletedAtGte) return []*entity.BackupSetting{}, nil }) slots := newBackupSyncSlots(cfg, backupSettingRepo, nil, nil, nil, nil, nil, nil) err := slots.rebuildWithoutLock(t.Context()) require.NoError(t, err) slots.isInit = true // save statues for latter assert clonedBackupSettings, err := deepClone(slots.backupSettings) require.NoError(t, err) clonedClusterSlots, err := deepClone(slots.clusterSlots) require.NoError(t, err) clonedSlots, err := deepClone(slots.slots) require.NoError(t, err) err = slots.update(t.Context()) require.NoError(t, err) // assert the slots assert.Equal(t, clonedBackupSettings, slots.backupSettings) assert.Equal(t, clonedClusterSlots, slots.clusterSlots) assert.Equal(t, clonedSlots, slots.slots) }) t.Run("one updated, one deleted, one new created", func(t *testing.T) { bs11, _ := deepClone(bs1) bs11.Schedule.Type = "per-hour" bs11.Schedule.At = 0 bs11.State = schema.BackupSettingStateCreated bs22, _ := deepClone(bs2) bs22.State = schema.BackupSettingStateDeleted bs4.State = schema.BackupSettingStateCreated ctrl := gomock.NewController(t) defer ctrl.Finish() backupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) // for building the slots backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), &brrepo.QueryBackupSettingFilter{ States: []schema.BackupSettingState{schema.BackupSettingStateCreated}, }, nil).Return([]*entity.BackupSetting{bs1, bs2, bs3}, nil) // for updating the slots - first call for recently schedule changed backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), gomock.Any(), nil).DoAndReturn( func(ctx context.Context, filter *brrepo.QueryBackupSettingFilter, page *pagination.Page) ([]*entity.BackupSetting, bizerr.BizError) { // Verify the filter for recently schedule changed backup settings assert.Equal(t, []schema.BackupSettingState{schema.BackupSettingStateCreated}, filter.States) assert.NotNil(t, filter.LastScheduleUpdatedAtGte) return []*entity.BackupSetting{bs11, bs4}, nil }) // for updating the slots - second call for recently deleted backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), gomock.Any(), nil).DoAndReturn( func(ctx context.Context, filter *brrepo.QueryBackupSettingFilter, page *pagination.Page) ([]*entity.BackupSetting, bizerr.BizError) { // Verify the filter reconciles every deleted setting still present in this process's cache. assert.Equal(t, schema.BackupSettingStatesForDeletedCluster, filter.States) assert.ElementsMatch(t, []uint64{1, 2, 3, 4}, filter.ClusterIds) assert.Nil(t, filter.DeletedAtGte) return []*entity.BackupSetting{bs22}, nil }) slots := newBackupSyncSlots(cfg, backupSettingRepo, nil, nil, nil, nil, nil, nil) err := slots.rebuildWithoutLock(t.Context()) require.NoError(t, err) slots.isInit = true // save statues for latter assert clonedBackupSettings, err := deepClone(slots.backupSettings) require.NoError(t, err) clonedClusterSlots, err := deepClone(slots.clusterSlots) require.NoError(t, err) clonedSlots, err := deepClone(slots.slots) require.NoError(t, err) err = slots.update(t.Context()) require.NoError(t, err) // assert the slots assert.NotEqual(t, clonedBackupSettings, slots.backupSettings) assert.NotEqual(t, clonedClusterSlots, slots.clusterSlots) assert.NotEqual(t, clonedSlots, slots.slots) // assert the slots changes assert.Equal(t, map[uint64]*entity.BackupSetting{1: bs11, 3: bs3, 4: bs4}, slots.backupSettings) // bs11: per-hour, 0, shared pool 1 // bs3: per-minute, 0, shared pool 2 // bs4: per-minute, 0, shared pool 2 // assert the cluster slots cluster1Slots := []int{ 0, 60, 120, 180, 240, 300, 360, 420, 480, 540, 600, 660, 720, 780, 840, 900, 960, 1020, 1080, 1140, 1200, 1260, 1320, 1380, } assert.Equal(t, cluster1Slots, slots.clusterSlots[1]) cluster3Slots := []int{} cluster4Slots := []int{} for i := range 24 * 60 { cluster3Slots = append(cluster3Slots, i) cluster4Slots = append(cluster4Slots, i) } assert.Equal(t, cluster3Slots, slots.clusterSlots[3]) assert.Equal(t, cluster4Slots, slots.clusterSlots[4]) // overlap slot with 3 clusters for _, slot := range cluster1Slots { assert.Len(t, slots.slots[slot], 2) // two shared pools assert.Len(t, slots.slots[slot][1], 1) // one clusters assert.Equal(t, map[uint64]bool{1: true}, slots.slots[slot][1]) assert.Len(t, slots.slots[slot][2], 2) // two cluster assert.Equal(t, map[uint64]bool{3: true, 4: true}, slots.slots[slot][2]) } // overlap slot with 2 clusters for _, slot := range cluster3Slots { if slot%60 == 0 { continue } assert.Len(t, slots.slots[slot], 1) assert.Len(t, slots.slots[slot][2], 2) assert.Equal(t, map[uint64]bool{3: true, 4: true}, slots.slots[slot][2]) } }) t.Run("batches deleted cached setting reconciliation", func(t *testing.T) { ctrl := gomock.NewController(t) backupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) cachedSettingCount := deletedSettingReconciliationBatchSize + 1 backupSettings := make(map[uint64]*entity.BackupSetting, cachedSettingCount) clusterSlots := make(map[uint64][]int, cachedSettingCount) for i := 1; i <= cachedSettingCount; i++ { clusterID := uint64(i) backupSettings[clusterID] = &entity.BackupSetting{BackupSetting: &schema.BackupSetting{ ClusterId: clusterID, State: schema.BackupSettingStateCreated, }} clusterSlots[clusterID] = nil } queriedClusterIDs := make(map[uint64]struct{}, cachedSettingCount) deletedClusterIDs := make([]uint64, 0, 2) gomock.InOrder( backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), gomock.Any(), nil).DoAndReturn( func(_ context.Context, filter *brrepo.QueryBackupSettingFilter, _ *pagination.Page) ([]*entity.BackupSetting, bizerr.BizError) { require.NotNil(t, filter.LastScheduleUpdatedAtGte) return nil, nil }, ), backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), gomock.Any(), nil).DoAndReturn( func(_ context.Context, filter *brrepo.QueryBackupSettingFilter, _ *pagination.Page) ([]*entity.BackupSetting, bizerr.BizError) { require.NotEmpty(t, filter.ClusterIds) require.LessOrEqual(t, len(filter.ClusterIds), deletedSettingReconciliationBatchSize) require.Equal(t, schema.BackupSettingStatesForDeletedCluster, filter.States) for _, clusterID := range filter.ClusterIds { queriedClusterIDs[clusterID] = struct{}{} } deletedClusterID := filter.ClusterIds[0] deletedClusterIDs = append(deletedClusterIDs, deletedClusterID) return []*entity.BackupSetting{{BackupSetting: &schema.BackupSetting{ ClusterId: deletedClusterID, State: schema.BackupSettingStateDeleted, }}}, nil }, ).Times(2), ) slots := &backupSyncSlots{ cfg: &config.SyncBackupsConfig{ LoadScheduleChangeInterval: 5 * time.Minute, }, backupSettingRepo: backupSettingRepo, lock: &sync.Mutex{}, isInit: true, slots: make([]map[uint64]map[uint64]bool, MinuteCountOfADay), clusterSlots: clusterSlots, backupSettings: backupSettings, } require.NoError(t, slots.update(t.Context())) require.Len(t, queriedClusterIDs, cachedSettingCount) require.Len(t, deletedClusterIDs, 2) for _, clusterID := range deletedClusterIDs { require.NotContains(t, slots.backupSettings, clusterID) } require.Len(t, slots.backupSettings, cachedSettingCount-len(deletedClusterIDs)) }) t.Run("query backup setting failed - first call", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() backupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) // for building the slots backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), &brrepo.QueryBackupSettingFilter{ States: []schema.BackupSettingState{schema.BackupSettingStateCreated}, }, nil).Return([]*entity.BackupSetting{bs1, bs2, bs3}, nil) // for updating the slots - first call fails backupSettingRepo.EXPECT(). UnscopedQuery(t.Context(), gomock.Any(), nil). Return(nil, bizerr.New(bizerr.InternalErrCodeCustomEnd, "failed to query backup setting")) slots := newBackupSyncSlots(cfg, backupSettingRepo, nil, nil, nil, nil, nil, nil) err := slots.rebuildWithoutLock(t.Context()) require.NoError(t, err) slots.isInit = true err = slots.update(t.Context()) require.Error(t, err) assert.Contains(t, err.Error(), "list recently schedule changed backup settings failed") }) t.Run("query backup setting failed - second call", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() backupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) // for building the slots backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), &brrepo.QueryBackupSettingFilter{ States: []schema.BackupSettingState{schema.BackupSettingStateCreated}, }, nil).Return([]*entity.BackupSetting{bs1, bs2, bs3}, nil).AnyTimes() // for updating the slots - first call succeeds backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), gomock.Any(), nil).DoAndReturn( func(ctx context.Context, filter *brrepo.QueryBackupSettingFilter, page *pagination.Page) ([]*entity.BackupSetting, bizerr.BizError) { assert.Equal(t, []schema.BackupSettingState{schema.BackupSettingStateCreated}, filter.States) assert.NotNil(t, filter.LastScheduleUpdatedAtGte) return []*entity.BackupSetting{}, nil }) // for updating the slots - second call fails backupSettingRepo.EXPECT(). UnscopedQuery(t.Context(), gomock.Any(), nil). Return(nil, bizerr.New(bizerr.InternalErrCodeCustomEnd, "failed to query deleted backup setting")) slots := newBackupSyncSlots(cfg, backupSettingRepo, nil, nil, nil, nil, nil, nil) err := slots.rebuildWithoutLock(t.Context()) require.NoError(t, err) slots.isInit = true err = slots.update(t.Context()) require.Error(t, err) assert.Contains(t, err.Error(), "list deleted cached backup settings failed") }) } func TestWorkerSyncBackupsReconcilesDeletedCacheBeforeSync(t *testing.T) { ctrl := gomock.NewController(t) backupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) cfg := &config.SyncBackupsConfig{ Enable: true, LoadScheduleChangeInterval: 5 * time.Minute, } setting := &entity.BackupSetting{BackupSetting: &schema.BackupSetting{ ClusterId: 100, SharedPoolId: 123, State: schema.BackupSettingStateCreated, }} deletedSetting, cloneErr := deepClone(setting) require.NoError(t, cloneErr) deletedSetting.State = schema.BackupSettingStateDeleted backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), gomock.Any(), nil).DoAndReturn( func(_ context.Context, filter *brrepo.QueryBackupSettingFilter, _ *pagination.Page) ([]*entity.BackupSetting, bizerr.BizError) { assert.Equal(t, []schema.BackupSettingState{schema.BackupSettingStateCreated}, filter.States) return nil, nil }) backupSettingRepo.EXPECT().UnscopedQuery(t.Context(), gomock.Any(), nil).DoAndReturn( func(_ context.Context, filter *brrepo.QueryBackupSettingFilter, _ *pagination.Page) ([]*entity.BackupSetting, bizerr.BizError) { assert.Equal(t, schema.BackupSettingStatesForDeletedCluster, filter.States) assert.Equal(t, []uint64{100}, filter.ClusterIds) return []*entity.BackupSetting{deletedSetting}, nil }) slots := make([]map[uint64]map[uint64]bool, MinuteCountOfADay) slots[60] = map[uint64]map[uint64]bool{123: {100: true}} syncSlots := &backupSyncSlots{ cfg: cfg, backupSettingRepo: backupSettingRepo, lock: &sync.Mutex{}, isInit: true, backupSettings: map[uint64]*entity.BackupSetting{100: setting}, clusterSlots: map[uint64][]int{100: {60}}, slots: slots, } w := &worker{ cfg: &config.BackupConfig{SyncConfig: *cfg}, backupSyncSlots: syncSlots, } require.NoError(t, w.SyncBackups(t.Context())) assert.NotContains(t, syncSlots.backupSettings, uint64(100)) assert.NotContains(t, syncSlots.clusterSlots, uint64(100)) assert.NotContains(t, syncSlots.slots[60][123], uint64(100)) } func TestBackupSyncSlots_calculateToBeSyncedUnit(t *testing.T) { t.Run("no minutes to be synced", func(t *testing.T) { slots := &backupSyncSlots{ slots: make([]map[uint64]map[uint64]bool, MinuteCountOfADay), backupSettings: make(map[uint64]*entity.BackupSetting), } now := time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC) result := slots.calculateToBeSyncedUnit(t.Context(), []int{}, now) assert.Empty(t, result) }) t.Run("minutes with no clusters in slots", func(t *testing.T) { slots := &backupSyncSlots{ slots: make([]map[uint64]map[uint64]bool, MinuteCountOfADay), backupSettings: make(map[uint64]*entity.BackupSetting), } // Initialize empty slots for i := range slots.slots { slots.slots[i] = make(map[uint64]map[uint64]bool) } now := time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC) result := slots.calculateToBeSyncedUnit(t.Context(), []int{60, 120}, now) assert.Empty(t, result) }) t.Run("minutes with clusters but all already synced", func(t *testing.T) { slots := &backupSyncSlots{ slots: make([]map[uint64]map[uint64]bool, MinuteCountOfADay), backupSettings: make(map[uint64]*entity.BackupSetting), } // Initialize slots with clusters slots.slots[60] = map[uint64]map[uint64]bool{ 1: {100: true, 101: true}, } slots.slots[120] = map[uint64]map[uint64]bool{ 2: {200: true}, } // Create backup settings that are already synced alreadySyncedTime := time.Date(2025, 1, 15, 3, 0, 0, 0, time.UTC) // 03:00 (after both slots) slots.backupSettings[100] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: &alreadySyncedTime, }, } slots.backupSettings[101] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: &alreadySyncedTime, }, } slots.backupSettings[200] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: &alreadySyncedTime, }, } now := time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC) var minutesToBeSynced []int for i := 60; i < 120; i++ { // from 01:00 to 02:00 minutesToBeSynced = append(minutesToBeSynced, i) } result := slots.calculateToBeSyncedUnit(t.Context(), minutesToBeSynced, now) assert.Empty(t, result) }) t.Run("minutes with clusters that need syncing", func(t *testing.T) { slots := &backupSyncSlots{ slots: make([]map[uint64]map[uint64]bool, MinuteCountOfADay), backupSettings: make(map[uint64]*entity.BackupSetting), } // Initialize slots with clusters slots.slots[60] = map[uint64]map[uint64]bool{ 1: {100: true, 101: true}, } slots.slots[120] = map[uint64]map[uint64]bool{ 2: {200: true}, } // Create backup settings - some synced, some not slotTime60 := time.Date(2025, 1, 15, 1, 0, 0, 0, time.UTC) // 01:00 slotTime120 := time.Date(2025, 1, 15, 2, 0, 0, 0, time.UTC) // 02:00 alreadySyncedTime := time.Date(2025, 1, 15, 0, 30, 0, 0, time.UTC) // 00:30 (before both slots) slots.backupSettings[100] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: &alreadySyncedTime, // Not synced for slot 60 }, } slots.backupSettings[101] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: &alreadySyncedTime, // Not synced for slot 60 }, } slots.backupSettings[200] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: &alreadySyncedTime, // Not synced for slot 120 }, } now := time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC) var minutesToBeSynced []int for i := 60; i <= 120; i++ { // from 01:00 to 02:00 minutesToBeSynced = append(minutesToBeSynced, i) } result := slots.calculateToBeSyncedUnit(t.Context(), minutesToBeSynced, now) assert.Len(t, result, 2) // Check first unit (slot 60) unit60 := result[0] assert.Equal(t, 60, unit60.slotIndex) assert.Equal(t, slotTime60, unit60.slotTime) assert.Len(t, unit60.sharedPoolClusters, 1) assert.Len(t, unit60.sharedPoolClusters[1], 2) assert.True(t, unit60.sharedPoolClusters[1][100]) assert.True(t, unit60.sharedPoolClusters[1][101]) // Check second unit (slot 120) unit120 := result[1] assert.Equal(t, 120, unit120.slotIndex) assert.Equal(t, slotTime120, unit120.slotTime) assert.Len(t, unit120.sharedPoolClusters, 1) assert.Len(t, unit120.sharedPoolClusters[2], 1) assert.True(t, unit120.sharedPoolClusters[2][200]) }) t.Run("minutes with mixed sync status", func(t *testing.T) { slots := &backupSyncSlots{ slots: make([]map[uint64]map[uint64]bool, MinuteCountOfADay), backupSettings: make(map[uint64]*entity.BackupSetting), } // Initialize slots with clusters slots.slots[60] = map[uint64]map[uint64]bool{ 1: {100: true, 101: true, 102: true}, } // Create backup settings - mixed sync status slotTime60 := time.Date(2025, 1, 15, 1, 0, 0, 0, time.UTC) // 01:00 alreadySyncedTime := time.Date(2025, 1, 15, 1, 30, 0, 0, time.UTC) // 01:30 (after slot) notSyncedTime := time.Date(2025, 1, 15, 0, 30, 0, 0, time.UTC) // 00:30 (before slot) slots.backupSettings[100] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: &alreadySyncedTime, // Already synced }, } slots.backupSettings[101] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: ¬SyncedTime, // Not synced }, } slots.backupSettings[102] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: nil, // No last backup time }, } now := time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC) var minutesToBeSynced []int for i := 60; i < 90; i++ { // from 01:00 to 01:30 minutesToBeSynced = append(minutesToBeSynced, i) } result := slots.calculateToBeSyncedUnit(t.Context(), minutesToBeSynced, now) assert.Len(t, result, 1) // Check unit (slot 60) unit60 := result[0] assert.Equal(t, 60, unit60.slotIndex) assert.Equal(t, slotTime60, unit60.slotTime) assert.Len(t, unit60.sharedPoolClusters, 1) assert.Len(t, unit60.sharedPoolClusters[1], 2) // Only 101 and 102, not 100 assert.False(t, unit60.sharedPoolClusters[1][100]) // Should not be included assert.True(t, unit60.sharedPoolClusters[1][101]) // Should be included assert.True(t, unit60.sharedPoolClusters[1][102]) // Should be included }) t.Run("negative minute index", func(t *testing.T) { slots := &backupSyncSlots{ slots: make([]map[uint64]map[uint64]bool, MinuteCountOfADay), backupSettings: make(map[uint64]*entity.BackupSetting), } // Initialize slots with clusters at the adjusted index negativeMinute := -30 adjustedIndex := negativeMinute + MinuteCountOfADay // -30 + 1440 = 1410 slots.slots[adjustedIndex] = map[uint64]map[uint64]bool{ 1: {100: true}, } // Create backup setting that needs syncing slotTime := time.Date(2025, 1, 14, 23, 30, 0, 0, time.UTC) // 23:30 - yesterday notSyncedTime := time.Date(2025, 1, 14, 23, 0, 0, 0, time.UTC) // 23:00 (before slot) - yesterday slots.backupSettings[100] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: ¬SyncedTime, }, } now := time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC) var minutesToBeSynced []int for i := negativeMinute; i < 12*60; i++ { // from 23:30(yesterday) to 12:00(today) minutesToBeSynced = append(minutesToBeSynced, i) } result := slots.calculateToBeSyncedUnit(t.Context(), minutesToBeSynced, now) assert.Len(t, result, 1) // Check unit unit := result[0] assert.Equal(t, adjustedIndex, unit.slotIndex) assert.Equal(t, slotTime, unit.slotTime) assert.Len(t, unit.sharedPoolClusters, 1) assert.Len(t, unit.sharedPoolClusters[1], 1) assert.True(t, unit.sharedPoolClusters[1][100]) }) t.Run("multiple shared pools in same slot", func(t *testing.T) { slots := &backupSyncSlots{ slots: make([]map[uint64]map[uint64]bool, MinuteCountOfADay), backupSettings: make(map[uint64]*entity.BackupSetting), } // Initialize slots with multiple shared pools slots.slots[60] = map[uint64]map[uint64]bool{ 1: {100: true, 101: true}, 2: {200: true}, } // Create backup settings that need syncing slotTime60 := time.Date(2025, 1, 15, 1, 0, 0, 0, time.UTC) // 01:00 notSyncedTime := time.Date(2025, 1, 15, 0, 30, 0, 0, time.UTC) // 00:30 (before slot) slots.backupSettings[100] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: ¬SyncedTime, }, } slots.backupSettings[101] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: ¬SyncedTime, }, } slots.backupSettings[200] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: ¬SyncedTime, }, } now := time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC) result := slots.calculateToBeSyncedUnit(t.Context(), []int{60}, now) assert.Len(t, result, 1) // Check unit (slot 60) unit60 := result[0] assert.Equal(t, 60, unit60.slotIndex) assert.Equal(t, slotTime60, unit60.slotTime) assert.Len(t, unit60.sharedPoolClusters, 2) assert.Len(t, unit60.sharedPoolClusters[1], 2) assert.True(t, unit60.sharedPoolClusters[1][100]) assert.True(t, unit60.sharedPoolClusters[1][101]) assert.Len(t, unit60.sharedPoolClusters[2], 1) assert.True(t, unit60.sharedPoolClusters[2][200]) }) t.Run("backup setting not found", func(t *testing.T) { slots := &backupSyncSlots{ slots: make([]map[uint64]map[uint64]bool, MinuteCountOfADay), backupSettings: make(map[uint64]*entity.BackupSetting), } // Initialize slots with clusters slots.slots[60] = map[uint64]map[uint64]bool{ 1: {100: true, 101: true}, } // Only create backup setting for cluster 100, not 101 slotTime60 := time.Date(2025, 1, 15, 1, 0, 0, 0, time.UTC) // 01:00 notSyncedTime := time.Date(2025, 1, 15, 0, 30, 0, 0, time.UTC) // 00:30 (before slot) slots.backupSettings[100] = &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ LastAutoBackupAt: ¬SyncedTime, }, } // Note: backupSettings[101] is not set now := time.Date(2025, 1, 15, 12, 0, 0, 0, time.UTC) result := slots.calculateToBeSyncedUnit(t.Context(), []int{60}, now) assert.Len(t, result, 1) // Check unit (slot 60) - should only include cluster 100 unit60 := result[0] assert.Equal(t, 60, unit60.slotIndex) assert.Equal(t, slotTime60, unit60.slotTime) assert.Len(t, unit60.sharedPoolClusters, 1) assert.Len(t, unit60.sharedPoolClusters[1], 1) assert.True(t, unit60.sharedPoolClusters[1][100]) assert.False(t, unit60.sharedPoolClusters[1][101]) // Should not be included }) } func TestBackupSyncSlots_fetchSharedPoolBackups(t *testing.T) { t.Run("successful fetch with single cluster", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() // Setup mocks clusterRepo := mockclusterrepo.NewMockRepo(ctrl) regionRepo := mockregionrepo.NewMockRepo(ctrl) regionalserverCli := mockregionalserver.NewMockClient(ctrl) // Create test data sharedPoolId := uint64(123) clusterIds := map[uint64]bool{100: true} toBeSyncedUnit := &toBeSyncedUnit{ slotIndex: 60, slotTime: time.Date(2025, 1, 15, 1, 0, 0, 0, time.UTC), } // Mock region region := &entity.Region{} cluster := &entity.Cluster{RegionId: 1} // Mock response expectedResponse := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-0100", BackupTs: 1705287600, BackupTime: "2025-01-15T01:00:00.000Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 100, }, }, }, }, } // Set expectations clusterRepo.EXPECT().Get(t.Context(), uint64(100), true).Return(cluster, nil) regionRepo.EXPECT().Get(t.Context(), uint64(1)).Return(region, nil) regionalserverCli.EXPECT().ListLightweightBackups( t.Context(), region, &clusterngv1beta.ListLightweightBackupsRequest{ SharedPoolId: "123", FromPrefix: "20250115/0058", LoadDetails: true, MaxCount: 10, }, ).Return(expectedResponse, nil) // Create backupSyncSlots instance slots := &backupSyncSlots{ clusterRepo: clusterRepo, regionRepo: regionRepo, regionalserverCli: regionalserverCli, } // Call the function result, err := slots.fetchSharedPoolBackups(t.Context(), sharedPoolId, clusterIds, toBeSyncedUnit) // Assertions require.NoError(t, err) assert.Equal(t, expectedResponse, result) assert.Len(t, result.Backups, 1) assert.Equal(t, "backup-20250115-0100", result.Backups[0].Name) }) t.Run("cluster not found", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() clusterRepo := mockclusterrepo.NewMockRepo(ctrl) // Create test data sharedPoolId := uint64(123) clusterIds := map[uint64]bool{100: true} toBeSyncedUnit := &toBeSyncedUnit{ slotIndex: 60, slotTime: time.Date(2025, 1, 15, 1, 0, 0, 0, time.UTC), } clusterRepo.EXPECT(). Get(t.Context(), uint64(100), true). Return(nil, bizerr.New(bizerr.ErrCodeRecordNotFound, "cluster not found")) // Create backupSyncSlots instance slots := &backupSyncSlots{clusterRepo: clusterRepo} // Call the function result, err := slots.fetchSharedPoolBackups(t.Context(), sharedPoolId, clusterIds, toBeSyncedUnit) // Assertions require.Error(t, err) assert.Nil(t, result) assert.Contains(t, err.Error(), "get cluster failed") }) t.Run("region not found", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() // Setup mocks clusterRepo := mockclusterrepo.NewMockRepo(ctrl) regionRepo := mockregionrepo.NewMockRepo(ctrl) // Create test data sharedPoolId := uint64(123) clusterIds := map[uint64]bool{100: true} toBeSyncedUnit := &toBeSyncedUnit{ slotIndex: 60, slotTime: time.Date(2025, 1, 15, 1, 0, 0, 0, time.UTC), } clusterRepo.EXPECT().Get(t.Context(), uint64(100), true).Return(&entity.Cluster{RegionId: 1}, nil) // Set expectations regionRepo.EXPECT(). Get(t.Context(), uint64(1)). Return(nil, bizerr.New(bizerr.ErrCodeRecordNotFound, "region not found")) // Create backupSyncSlots instance slots := &backupSyncSlots{ clusterRepo: clusterRepo, regionRepo: regionRepo, } // Call the function result, err := slots.fetchSharedPoolBackups(t.Context(), sharedPoolId, clusterIds, toBeSyncedUnit) // Assertions require.Error(t, err) assert.Nil(t, result) assert.Contains(t, err.Error(), "get region failed") }) t.Run("regional server call failed", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() // Setup mocks clusterRepo := mockclusterrepo.NewMockRepo(ctrl) regionRepo := mockregionrepo.NewMockRepo(ctrl) regionalserverCli := mockregionalserver.NewMockClient(ctrl) // Create test data sharedPoolId := uint64(123) clusterIds := map[uint64]bool{100: true} toBeSyncedUnit := &toBeSyncedUnit{ slotIndex: 60, slotTime: time.Date(2025, 1, 15, 1, 0, 0, 0, time.UTC), } // Mock region region := &entity.Region{} cluster := &entity.Cluster{RegionId: 1} // Set expectations clusterRepo.EXPECT().Get(t.Context(), uint64(100), true).Return(cluster, nil) regionRepo.EXPECT().Get(t.Context(), uint64(1)).Return(region, nil) regionalserverCli.EXPECT().ListLightweightBackups( t.Context(), region, &clusterngv1beta.ListLightweightBackupsRequest{ SharedPoolId: "123", FromPrefix: "20250115/0058", LoadDetails: true, MaxCount: 10, }, ).Return(nil, bizerr.New(bizerr.InternalErrCodeCustomEnd, "regional server error")) // Create backupSyncSlots instance slots := &backupSyncSlots{ clusterRepo: clusterRepo, regionRepo: regionRepo, regionalserverCli: regionalserverCli, } // Call the function result, err := slots.fetchSharedPoolBackups(t.Context(), sharedPoolId, clusterIds, toBeSyncedUnit) // Assertions require.Error(t, err) assert.Nil(t, result) assert.Contains(t, err.Error(), "list lightweight backups failed") }) t.Run("time formatting edge cases", func(t *testing.T) { testCases := []struct { name string slotTime time.Time expected string }{ { name: "midnight", slotTime: time.Date(2025, 1, 15, 0, 0, 0, 0, time.UTC), expected: "20250114/2358", }, { name: "end of day", slotTime: time.Date(2025, 1, 15, 23, 59, 0, 0, time.UTC), expected: "20250115/2357", }, { name: "single digit hour and minute", slotTime: time.Date(2025, 1, 15, 9, 5, 0, 0, time.UTC), expected: "20250115/0903", }, { name: "year end", slotTime: time.Date(2025, 12, 31, 12, 30, 0, 0, time.UTC), expected: "20251231/1228", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() // Setup mocks clusterRepo := mockclusterrepo.NewMockRepo(ctrl) regionRepo := mockregionrepo.NewMockRepo(ctrl) regionalserverCli := mockregionalserver.NewMockClient(ctrl) // Create test data sharedPoolId := uint64(123) clusterIds := map[uint64]bool{100: true} toBeSyncedUnit := &toBeSyncedUnit{ slotIndex: 60, slotTime: tc.slotTime, } // Mock region region := &entity.Region{} cluster := &entity.Cluster{RegionId: 1} // Mock response expectedResponse := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{}, } // Set expectations clusterRepo.EXPECT().Get(t.Context(), uint64(100), true).Return(cluster, nil) regionRepo.EXPECT().Get(t.Context(), uint64(1)).Return(region, nil) regionalserverCli.EXPECT().ListLightweightBackups( t.Context(), region, &clusterngv1beta.ListLightweightBackupsRequest{ SharedPoolId: "123", FromPrefix: tc.expected, LoadDetails: true, MaxCount: 10, }, ).Return(expectedResponse, nil) // Create backupSyncSlots instance slots := &backupSyncSlots{ clusterRepo: clusterRepo, regionRepo: regionRepo, regionalserverCli: regionalserverCli, } // Call the function result, err := slots.fetchSharedPoolBackups(t.Context(), sharedPoolId, clusterIds, toBeSyncedUnit) // Assertions require.NoError(t, err) assert.Equal(t, expectedResponse, result) }) } }) t.Run("multiple clusters in shared pool", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() // Setup mocks clusterRepo := mockclusterrepo.NewMockRepo(ctrl) regionRepo := mockregionrepo.NewMockRepo(ctrl) regionalserverCli := mockregionalserver.NewMockClient(ctrl) // Create test data with multiple clusters sharedPoolId := uint64(123) clusterIds := map[uint64]bool{100: true, 101: true, 102: true} toBeSyncedUnit := &toBeSyncedUnit{ slotIndex: 60, slotTime: time.Date(2025, 1, 15, 1, 0, 0, 0, time.UTC), } // Mock region region := &entity.Region{} cluster := &entity.Cluster{RegionId: 1} // Mock response with multiple cluster backup info expectedResponse := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-0100", BackupTs: 1705287600, BackupTime: "2025-01-15T01:00:00.000Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 100, }, { ClusterId: "101", BackupSizeBytes: 100, }, { ClusterId: "102", BackupSizeBytes: 100, }, }, }, }, } // Set expectations clusterRepo.EXPECT().Get(t.Context(), gomock.Any(), true).Return(cluster, nil) regionRepo.EXPECT().Get(t.Context(), uint64(1)).Return(region, nil) regionalserverCli.EXPECT().ListLightweightBackups( t.Context(), region, &clusterngv1beta.ListLightweightBackupsRequest{ SharedPoolId: "123", FromPrefix: "20250115/0058", LoadDetails: true, MaxCount: 10, }, ).Return(expectedResponse, nil) // Create backupSyncSlots instance slots := &backupSyncSlots{ clusterRepo: clusterRepo, regionRepo: regionRepo, regionalserverCli: regionalserverCli, } // Call the function result, err := slots.fetchSharedPoolBackups(t.Context(), sharedPoolId, clusterIds, toBeSyncedUnit) // Assertions require.NoError(t, err) assert.Equal(t, expectedResponse, result) assert.Len(t, result.Backups, 1) assert.Len(t, result.Backups[0].ClusterBackupInfo, 3) }) } func TestIsBackupTimeMatchSlotTime(t *testing.T) { ctx := t.Context() t.Run("per-day schedule - should match the daily window", func(t *testing.T) { backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-day", At: 1 * 60, }, }, } slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) testCases := []struct { name string backupTime time.Time expected bool }{ { name: "same day, exact slot minute", backupTime: time.Date(2025, 1, 15, 14, 0, 15, 0, time.UTC), expected: true, }, { name: "same day, same hour, different minute", backupTime: time.Date(2025, 1, 15, 14, 15, 0, 0, time.UTC), expected: true, }, { name: "same day, midnight", backupTime: time.Date(2025, 1, 15, 0, 0, 0, 0, time.UTC), expected: false, }, { name: "same day, end of day", backupTime: time.Date(2025, 1, 15, 23, 59, 59, 0, time.UTC), expected: true, }, { name: "next day slot boundary", backupTime: time.Date(2025, 1, 16, 14, 0, 0, 0, time.UTC), expected: false, }, { name: "different day after slot boundary", backupTime: time.Date(2025, 1, 16, 14, 30, 0, 0, time.UTC), expected: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := isBackupTimeMatchSlotTime(ctx, tc.backupTime, slotTime, backupSetting) assert.Equal(t, tc.expected, result) }) } }) t.Run("per-hour schedule - should match the hourly window", func(t *testing.T) { backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }, } slotTime := time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC) // 14:30 testCases := []struct { name string backupTime time.Time expected bool }{ { name: "same hour, exact slot minute", backupTime: time.Date(2025, 1, 15, 14, 30, 45, 0, time.UTC), // 14:30:45 expected: true, }, { name: "same hour, different minute", backupTime: time.Date(2025, 1, 15, 14, 15, 0, 0, time.UTC), // 14:15 expected: false, }, { name: "same hour, start of hour", backupTime: time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC), // 14:00 expected: false, }, { name: "same hour, after slot minute", backupTime: time.Date(2025, 1, 15, 14, 59, 59, 0, time.UTC), // 14:59 expected: true, }, { name: "different hour, same day", backupTime: time.Date(2025, 1, 15, 15, 30, 0, 0, time.UTC), // 15:30 expected: false, }, { name: "same hour, different day", backupTime: time.Date(2025, 1, 16, 14, 30, 0, 0, time.UTC), // 16th 14:30 expected: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := isBackupTimeMatchSlotTime(ctx, tc.backupTime, slotTime, backupSetting) assert.Equal(t, tc.expected, result) }) } }) t.Run("per-minute schedule - should match same minute", func(t *testing.T) { backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-minute", At: 0, }, }, } slotTime := time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC) // 14:30:00 testCases := []struct { name string backupTime time.Time expected bool }{ { name: "same minute, different second", backupTime: time.Date(2025, 1, 15, 14, 30, 45, 0, time.UTC), // 14:30:45 expected: true, }, { name: "same minute, start of minute", backupTime: time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC), // 14:30:00 expected: true, }, { name: "same minute, end of minute", backupTime: time.Date(2025, 1, 15, 14, 30, 59, 0, time.UTC), // 14:30:59 expected: true, }, { name: "different minute, same hour", backupTime: time.Date(2025, 1, 15, 14, 31, 0, 0, time.UTC), // 14:31:00 expected: false, }, { name: "different minute, different hour", backupTime: time.Date(2025, 1, 15, 15, 30, 0, 0, time.UTC), // 15:30:00 expected: false, }, { name: "same minute, different day", backupTime: time.Date(2025, 1, 16, 14, 30, 0, 0, time.UTC), // 14:30:00 expected: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := isBackupTimeMatchSlotTime(ctx, tc.backupTime, slotTime, backupSetting) assert.Equal(t, tc.expected, result) }) } }) t.Run("unknown schedule type - should return false", func(t *testing.T) { backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "unknown-type", At: 0, }, }, } slotTime := time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC) backupTime := time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC) result := isBackupTimeMatchSlotTime(ctx, backupTime, slotTime, backupSetting) assert.False(t, result) }) t.Run("timezone handling - should convert to UTC", func(t *testing.T) { backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }, } // Slot time in UTC slotTime := time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC) // 14:30 UTC testCases := []struct { name string backupTime time.Time expected bool }{ { name: "backup time in EST (UTC-5)", backupTime: time.Date( 2025, 1, 15, 9, 30, 0, 0, time.FixedZone("EST", -5*3600), ), // 09:30 EST = 14:30 UTC expected: true, }, { name: "backup time in PST (UTC-8)", backupTime: time.Date( 2025, 1, 15, 6, 30, 0, 0, time.FixedZone("PST", -8*3600), ), // 06:30 PST = 14:30 UTC expected: true, }, { name: "backup time in JST (UTC+9)", backupTime: time.Date( 2025, 1, 15, 23, 30, 0, 0, time.FixedZone("JST", 9*3600), ), // 23:30 JST = 14:30 UTC expected: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := isBackupTimeMatchSlotTime(ctx, tc.backupTime, slotTime, backupSetting) assert.Equal(t, tc.expected, result) }) } }) t.Run("edge cases - boundary conditions", func(t *testing.T) { backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-minute", At: 0, }, }, } // Test at minute boundary slotTime := time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC) // 14:30:00 testCases := []struct { name string backupTime time.Time expected bool }{ { name: "exact minute boundary", backupTime: time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC), expected: true, }, { name: "one second before minute boundary", backupTime: time.Date(2025, 1, 15, 14, 29, 59, 0, time.UTC), expected: false, }, { name: "one second after minute boundary", backupTime: time.Date(2025, 1, 15, 14, 30, 1, 0, time.UTC), expected: true, // Still within the same minute }, { name: "59 seconds after minute boundary", backupTime: time.Date(2025, 1, 15, 14, 30, 59, 0, time.UTC), expected: true, }, { name: "one minute after boundary", backupTime: time.Date(2025, 1, 15, 14, 31, 0, 0, time.UTC), expected: false, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := isBackupTimeMatchSlotTime(ctx, tc.backupTime, slotTime, backupSetting) assert.Equal(t, tc.expected, result) }) } }) t.Run("hour boundary edge cases", func(t *testing.T) { backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }, } // Test at hour boundary slotTime := time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC) // 14:30 testCases := []struct { name string backupTime time.Time expected bool }{ { name: "exact configured slot", backupTime: time.Date(2025, 1, 15, 14, 30, 0, 0, time.UTC), expected: true, }, { name: "exact hour boundary", backupTime: time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC), expected: false, }, { name: "one minute before hour boundary", backupTime: time.Date(2025, 1, 15, 13, 59, 0, 0, time.UTC), expected: false, }, { name: "one minute after hour boundary", backupTime: time.Date(2025, 1, 15, 14, 1, 0, 0, time.UTC), expected: false, }, { name: "29 minutes after configured slot", backupTime: time.Date(2025, 1, 15, 14, 59, 0, 0, time.UTC), expected: true, }, { name: "30 minutes after configured slot", backupTime: time.Date(2025, 1, 15, 15, 0, 0, 0, time.UTC), expected: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { result := isBackupTimeMatchSlotTime(ctx, tc.backupTime, slotTime, backupSetting) assert.Equal(t, tc.expected, result) }) } }) } func TestAssembleBackupRecords(t *testing.T) { ctx := t.Context() t.Run("successful assembly with multiple clusters", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true, 101: true, 102: true} // Create backup settings backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, TenantId: 1000, ServicePlan: "Starter", RegionId: "us-west-2", ClusterVersion: "v7.5.0", SharedPoolId: 123, CloudProvider: "aws", Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 7, }, }, }, 101: { BackupSetting: &schema.BackupSetting{ ClusterId: 101, TenantId: 1001, ServicePlan: "Premium", RegionId: "us-west-2", ClusterVersion: "v7.5.0", SharedPoolId: 123, CloudProvider: "aws", Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 14, }, }, }, 102: { BackupSetting: &schema.BackupSetting{ ClusterId: 102, TenantId: 1002, ServicePlan: "Essential", RegionId: "us-west-2", ClusterVersion: "v7.5.0", SharedPoolId: 123, CloudProvider: "aws", Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 30, }, }, }, } // Create backup response backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ PdClusterId: 123, DfsPrefix: "backup-prefix", DfsBucket: "backup-bucket", DfsRegion: "us-west-2", Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-1400", BackupTs: 1705320000, BackupTime: "2025-01-15T14:00:00.000Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 1024 * 1024 * 100, // 100MB }, { ClusterId: "101", BackupSizeBytes: 1024 * 1024 * 200, // 200MB }, { ClusterId: "102", BackupSizeBytes: 1024 * 1024 * 300, // 300MB }, }, }, }, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Assertions assert.Len(t, records, 3) assert.Empty(t, missingClusters) // Check cluster 100 record100 := records[100] require.NotNil(t, record100) assert.Equal(t, uint64(100), record100.ClusterId) assert.Equal(t, uint64(1000), record100.TenantId) assert.Equal(t, "Starter", record100.ServicePlan) assert.Equal(t, "auto-backup-202501151400", record100.DisplayName) assert.Equal(t, uint64(1024*1024*100), record100.BackupSizeBytes) assert.Equal(t, schema.BackupStateSuccess, record100.State) assert.Equal(t, schema.BackupTypeSnapshot, record100.Type) assert.Equal(t, schema.BackupTriggerTypeAuto, record100.TriggerType) assert.Equal(t, uint64(789), record100.KernelBackupId) assert.Equal(t, "backup-20250115-1400", record100.KernelBackupName) assert.Equal(t, uint64(123), record100.PdClusterId) assert.Equal(t, "backup-prefix", record100.DfsPrefix) assert.Equal(t, "backup-bucket", record100.DfsBucket) assert.Equal(t, "us-west-2", record100.DfsRegion) assert.Equal(t, "v7.5.0", record100.ClusterVersion) assert.Equal(t, uint64(123), record100.SharedPoolId) assert.Equal(t, "aws", record100.CloudProvider) // Check expiration time for cluster 100 (7 days retention) expectedExpiration100 := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC).Add(7 * 24 * time.Hour) assert.Equal(t, expectedExpiration100.Unix(), record100.ExpirationTime.Unix()) // Check cluster 101 record101 := records[101] require.NotNil(t, record101) assert.Equal(t, uint64(101), record101.ClusterId) assert.Equal(t, uint64(1001), record101.TenantId) assert.Equal(t, "Premium", record101.ServicePlan) assert.Equal(t, uint64(1024*1024*200), record101.BackupSizeBytes) // Check expiration time for cluster 101 (14 days retention) expectedExpiration101 := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC).Add(14 * 24 * time.Hour) assert.Equal(t, expectedExpiration101.Unix(), record101.ExpirationTime.Unix()) // Check cluster 102 record102 := records[102] require.NotNil(t, record102) assert.Equal(t, uint64(102), record102.ClusterId) assert.Equal(t, uint64(1002), record102.TenantId) assert.Equal(t, "Essential", record102.ServicePlan) assert.Equal(t, uint64(1024*1024*300), record102.BackupSizeBytes) // Check expiration time for cluster 102 (30 days retention) expectedExpiration102 := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC).Add(30 * 24 * time.Hour) assert.Equal(t, expectedExpiration102.Unix(), record102.ExpirationTime.Unix()) }) t.Run("backup time doesn't match slot time - should create failed backup record", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) // 14:00 clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 7, }, }, }, } // Create backup response with different hour backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-1500", BackupTs: 1705320000, BackupTime: "2025-01-15T15:00:00.000Z", // 15:00, different hour ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 1024 * 1024 * 100, }, }, }, }, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Assertions - should create failed backup record for missing cluster assert.Len(t, records, 1) assert.Len(t, missingClusters, 1) assert.Contains(t, missingClusters, uint64(100)) // Verify failed backup record failedRecord := records[100] require.NotNil(t, failedRecord) assert.Equal(t, schema.BackupStateFailed, failedRecord.State) assert.Equal(t, "backup failed", failedRecord.Description) assert.Equal(t, slotTime, failedRecord.BackupTime) assert.Equal(t, uint64(100), failedRecord.ClusterId) }) t.Run("backup setting not found - should skip and log error", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true, 101: true} // Only provide backup setting for cluster 100, not 101 backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }, }, } backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-1400", BackupTs: 1705320000, BackupTime: "2025-01-15T14:00:00.000Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 1024 * 1024 * 100, }, { ClusterId: "101", BackupSizeBytes: 1024 * 1024 * 200, }, }, }, }, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Assertions assert.Len(t, records, 1) assert.Contains(t, records, uint64(100)) assert.Len(t, missingClusters, 1) assert.Contains(t, missingClusters, uint64(101)) }) t.Run("invalid backup time format - should create failed backup record for missing cluster", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 7, }, }, }, } // Create backup response with invalid time format backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-1400", BackupTs: 1705320000, BackupTime: "invalid-time-format", // Invalid format ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 1024 * 1024 * 100, }, }, }, }, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Assertions - should create failed backup record for missing cluster assert.Len(t, records, 1) assert.Len(t, missingClusters, 1) assert.Contains(t, missingClusters, uint64(100)) // Verify failed backup record failedRecord := records[100] require.NotNil(t, failedRecord) assert.Equal(t, schema.BackupStateFailed, failedRecord.State) assert.Equal(t, "backup failed", failedRecord.Description) assert.Equal(t, slotTime, failedRecord.BackupTime) }) t.Run("cluster not in backup info - should create failed backup record for missing cluster", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true, 101: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 7, }, }, }, 101: { BackupSetting: &schema.BackupSetting{ ClusterId: 101, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 7, }, }, }, } // Create backup response with only cluster 100 backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-1400", BackupTs: 1705320000, BackupTime: "2025-01-15T14:00:00.000Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 1024 * 1024 * 100, }, // Note: cluster 101 is missing from backup info }, }, }, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Assertions - should now have both records (100 success, 101 failed) assert.Len(t, records, 2) assert.Contains(t, records, uint64(100)) assert.Contains(t, records, uint64(101)) assert.Len(t, missingClusters, 1) assert.Contains(t, missingClusters, uint64(101)) // Verify successful backup for cluster 100 assert.Equal(t, schema.BackupStateSuccess, records[100].State) // Verify failed backup for cluster 101 assert.Equal(t, schema.BackupStateFailed, records[101].State) assert.Equal(t, "backup failed", records[101].Description) }) t.Run("multiple backups sorted by time - should use earliest matching backup", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }, }, } // Create backup response with multiple backups (not sorted) backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-1405", BackupTs: 1705320300, // Later timestamp BackupTime: "2025-01-15T14:05:00.000Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 1024 * 1024 * 200, }, }, }, { Id: 788, Name: "backup-20250115-1400", BackupTs: 1705320000, // Earlier timestamp BackupTime: "2025-01-15T14:00:00.000Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 1024 * 1024 * 100, }, }, }, }, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Assertions assert.Len(t, records, 1) assert.Empty(t, missingClusters) // Should use the earliest backup (ID 788) record := records[100] assert.Equal(t, uint64(788), record.KernelBackupId) assert.Equal(t, "backup-20250115-1400", record.KernelBackupName) assert.Equal(t, uint64(1024*1024*100), record.BackupSizeBytes) }) t.Run("hourly backup can match later backup time in slot window", func(t *testing.T) { slotTime := time.Date(2025, 1, 15, 9, 0, 0, 0, time.UTC) backupTime := time.Date(2025, 1, 15, 9, 1, 15, 0, time.UTC) clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, DailyBackupMinuteOfDay: 0, ZeroClockRetentionDays: 33, NonZeroClockRetentionDays: 7, }, BackupPolicy: schema.BackupPolicyNameTIER1, }, }, } backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "20250115/090100.meta", BackupTs: 1705323675, BackupTime: backupTime.Format(time.RFC3339Nano), ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{{ ClusterId: "100", BackupSizeBytes: 1024, }}, }, }, } records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) require.Empty(t, missingClusters) record := records[100] require.NotNil(t, record) assert.Equal(t, schema.BackupStateSuccess, record.State) assert.Equal(t, backupTime, record.BackupTime) assert.Equal(t, "auto-backup-202501150901", record.DisplayName) assert.Equal(t, uint64(1705323675), record.BackupTs) assert.Equal(t, "20250115/090100.meta", record.KernelBackupName) assert.Equal(t, backupTime.Add(7*24*time.Hour), record.ExpirationTime) }) t.Run("hourly backup excludes next slot boundary", func(t *testing.T) { slotTime := time.Date(2025, 1, 15, 9, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }, }, } backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "20250115/100000.meta", BackupTs: 1705327200, BackupTime: "2025-01-15T10:00:00Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{{ ClusterId: "100", }}, }, }, } records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) require.Len(t, missingClusters, 1) assert.Equal(t, schema.BackupStateFailed, records[100].State) }) t.Run("daily flexible backup can match later backup time in daily window", func(t *testing.T) { slotTime := time.Date(2025, 1, 15, 9, 0, 0, 0, time.UTC) backupTime := time.Date(2025, 1, 15, 9, 1, 15, 0, time.UTC) clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Mode: schema.BackupModeFlexibleRetention, Schedule: schema.AutoSchedule{ Type: "per-day", At: 9 * 60, DailyBackupMinuteOfDay: 9 * 60, ZeroClockRetentionDays: 20, NonZeroClockRetentionDays: 3, }, BackupPolicy: schema.BackupPolicyNameTIER1, }, }, } backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "20250115/090100.meta", BackupTs: 1705323675, BackupTime: backupTime.Format(time.RFC3339Nano), ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{{ ClusterId: "100", }}, }, }, } records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) require.Empty(t, missingClusters) record := records[100] require.NotNil(t, record) assert.Equal(t, backupTime, record.BackupTime) assert.Equal(t, backupTime.Add(20*24*time.Hour), record.ExpirationTime) }) t.Run("per-minute backup still requires exact minute", func(t *testing.T) { slotTime := time.Date(2025, 1, 15, 9, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-minute", }, }, }, } backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "20250115/090115.meta", BackupTs: 1705323675, BackupTime: "2025-01-15T09:01:15Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{{ ClusterId: "100", }}, }, }, } records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) require.Len(t, missingClusters, 1) assert.Equal(t, schema.BackupStateFailed, records[100].State) }) t.Run("empty backup response - should create failed backup record for all missing clusters", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true, 101: true, 102: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 7, }, }, }, 101: { BackupSetting: &schema.BackupSetting{ ClusterId: 101, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 14, }, }, }, 102: { BackupSetting: &schema.BackupSetting{ ClusterId: 102, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 30, }, }, }, } // Create empty backup response backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{}, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Assertions - should create failed backup records for all missing clusters assert.Len(t, records, 3) assert.Len(t, missingClusters, 3) assert.Contains(t, missingClusters, uint64(100)) assert.Contains(t, missingClusters, uint64(101)) assert.Contains(t, missingClusters, uint64(102)) // Verify all records are failed backups for clusterId, record := range records { assert.Equal(t, schema.BackupStateFailed, record.State) assert.Equal(t, "backup failed", record.Description) assert.Equal(t, slotTime, record.BackupTime) assert.Equal(t, clusterId, record.ClusterId) } }) t.Run("zero backup size - should handle correctly", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }, }, } // Create backup response with zero size backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-1400", BackupTs: 1705320000, BackupTime: "2025-01-15T14:00:00.000Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: 0, // Zero size }, }, }, }, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Assertions assert.Len(t, records, 1) assert.Empty(t, missingClusters) record := records[100] assert.Equal(t, uint64(0), record.BackupSizeBytes) }) t.Run("large backup size - should handle correctly", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, }, }, }, } // Create backup response with large size largeSize := uint64(1024 * 1024 * 1024 * 10) // 10GB backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{ { Id: 789, Name: "backup-20250115-1400", BackupTs: 1705320000, BackupTime: "2025-01-15T14:00:00.000Z", ClusterBackupInfo: []*clusterngv1beta.ClusterBackupInfo{ { ClusterId: "100", BackupSizeBytes: largeSize, }, }, }, }, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Assertions assert.Len(t, records, 1) assert.Empty(t, missingClusters) record := records[100] assert.Equal(t, largeSize, record.BackupSizeBytes) }) t.Run("failed backup records should have correct retention days", func(t *testing.T) { // Setup test data with different retention days slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true, 101: true, 102: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 7, }, }, }, 101: { BackupSetting: &schema.BackupSetting{ ClusterId: 101, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 14, }, }, }, 102: { BackupSetting: &schema.BackupSetting{ ClusterId: 102, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 30, }, }, }, } // Create empty backup response backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{}, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Verify retention days for each failed backup record assert.Len(t, records, 3) assert.Len(t, missingClusters, 3) // Check cluster 100 (7 days retention) expectedExpiration100 := slotTime.Add(7 * 24 * time.Hour) assert.Equal(t, expectedExpiration100.Unix(), records[100].ExpirationTime.Unix()) // Check cluster 101 (14 days retention) expectedExpiration101 := slotTime.Add(14 * 24 * time.Hour) assert.Equal(t, expectedExpiration101.Unix(), records[101].ExpirationTime.Unix()) // Check cluster 102 (30 days retention) expectedExpiration102 := slotTime.Add(30 * 24 * time.Hour) assert.Equal(t, expectedExpiration102.Unix(), records[102].ExpirationTime.Unix()) }) t.Run("failed backup records should have all required fields", func(t *testing.T) { // Setup test data slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, TenantId: 1000, ServicePlan: "Premium", RegionId: "us-west-2", CloudProvider: "aws", ClusterVersion: "v7.5.0", SharedPoolId: 123, Schedule: schema.AutoSchedule{ Type: "per-day", At: 60, RetentionDays: 7, }, }, }, } // Create empty backup response backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{}, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Verify all required fields in failed backup record assert.Len(t, records, 1) assert.Len(t, missingClusters, 1) failedRecord := records[100] require.NotNil(t, failedRecord) // Verify basic fields assert.Equal(t, schema.BackupStateFailed, failedRecord.State) assert.Equal(t, schema.BackupTypeSnapshot, failedRecord.Type) assert.Equal(t, schema.BackupTriggerTypeAuto, failedRecord.TriggerType) assert.Equal(t, "backup failed", failedRecord.Description) assert.Equal(t, "auto-backup-202501151400", failedRecord.DisplayName) // Verify cluster-specific fields from backup setting assert.Equal(t, uint64(100), failedRecord.ClusterId) assert.Equal(t, uint64(1000), failedRecord.TenantId) assert.Equal(t, "Premium", failedRecord.ServicePlan) assert.Equal(t, "us-west-2", failedRecord.RegionId) assert.Equal(t, "aws", failedRecord.CloudProvider) assert.Equal(t, "v7.5.0", failedRecord.ClusterVersion) assert.Equal(t, uint64(123), failedRecord.SharedPoolId) // Verify time fields assert.Equal(t, slotTime, failedRecord.BackupTime) expectedExpiration := slotTime.Add(7 * 24 * time.Hour) assert.Equal(t, expectedExpiration.Unix(), failedRecord.ExpirationTime.Unix()) // Verify kernel backup fields are not set (since backup failed) assert.Equal(t, uint64(0), failedRecord.KernelBackupId) assert.Empty(t, failedRecord.KernelBackupName) assert.Equal(t, uint64(0), failedRecord.BackupSizeBytes) assert.NotEqual(t, uint64(0), failedRecord.BackupTs) assert.Equal(t, uint64(0), failedRecord.PdClusterId) assert.Empty(t, failedRecord.DfsPrefix) assert.Empty(t, failedRecord.DfsBucket) assert.Empty(t, failedRecord.DfsRegion) }) t.Run("missing backup setting should skip failed record creation", func(t *testing.T) { // Setup test data - cluster 101 has no backup setting slotTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) clusterIds := map[uint64]bool{100: true, 101: true} backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, Schedule: schema.AutoSchedule{ Type: "per-hour", At: 0, RetentionDays: 7, }, }, }, // Note: cluster 101 has no backup setting } // Create empty backup response backupResp := &clusterngv1beta.ListLightweightBackupsResponse{ Backups: []*clusterngv1beta.LightweightBackup{}, } // Call the function records, missingClusters := assembleBackupRecords(ctx, backupResp, clusterIds, backupSettings, slotTime) // Verify we have only 1 record (cluster 100) assert.Len(t, records, 1) assert.Contains(t, records, uint64(100)) assert.NotContains(t, records, uint64(101)) // Should not create failed record without backup setting assert.Len(t, missingClusters, 2) // Both clusters are in missing list assert.Contains(t, missingClusters, uint64(100)) assert.Contains(t, missingClusters, uint64(101)) }) } func TestSaveBackupSyncResult(t *testing.T) { ctx := t.Context() ctrl := gomock.NewController(t) defer ctrl.Finish() t.Run("real backup bypasses synthetic schedule and pause fences", func(t *testing.T) { mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) record := &entity.Backup{Backup: &schema.Backup{ ClusterId: 100, SharedPoolId: 123, BackupTime: backupTime, State: schema.BackupStateSuccess, }} setting := &entity.BackupSetting{BackupSetting: &schema.BackupSetting{ ClusterId: 100, SharedPoolId: 123, LastScheduleUpdatedAt: backupTime.Add(time.Hour), }} mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }) mockBackupRepo.EXPECT().Create(ctx, record).Return(nil) mockBackupSettingRepo.EXPECT().UpdateLastAutoBackupAt(ctx, uint64(100), &backupTime).Return(backupTime, nil) slots := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: map[uint64]*entity.BackupSetting{100: setting}, pauseFencer: func(context.Context, uint64, func(context.Context) error) (bool, error) { t.Fatal("a real Regional backup must not enter the synthetic-record fence") return false, nil }, } require.Empty(t, slots.saveBackupSyncResult(ctx, map[uint64]*entity.Backup{100: record})) require.Equal(t, &backupTime, setting.LastAutoBackupAt) }) t.Run("late real backup does not move watermark backwards", func(t *testing.T) { mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) newerWatermark := backupTime.Add(time.Hour) record := &entity.Backup{Backup: &schema.Backup{ ClusterId: 100, SharedPoolId: 123, BackupTime: backupTime, State: schema.BackupStateSuccess, }} setting := &entity.BackupSetting{BackupSetting: &schema.BackupSetting{ ClusterId: 100, SharedPoolId: 123, LastAutoBackupAt: &newerWatermark, }} mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }) mockBackupRepo.EXPECT().Create(ctx, record).Return(nil) mockBackupSettingRepo.EXPECT(). UpdateLastAutoBackupAt(ctx, uint64(100), &backupTime). Return(newerWatermark, nil) slots := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: map[uint64]*entity.BackupSetting{100: setting}, } require.Empty(t, slots.saveBackupSyncResult(ctx, map[uint64]*entity.Backup{100: record})) require.Equal(t, &newerWatermark, setting.LastAutoBackupAt) }) t.Run("successful save with multiple records", func(t *testing.T) { // Setup mocks mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) // Create test data backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) records := map[uint64]*entity.Backup{ 100: { Backup: &schema.Backup{ ClusterId: 100, TenantId: 1000, ServicePlan: "Starter", DisplayName: "auto-backup-202501151400", BackupTime: backupTime, BackupSizeBytes: 1024 * 1024 * 100, State: schema.BackupStateSuccess, Type: schema.BackupTypeSnapshot, TriggerType: schema.BackupTriggerTypeAuto, KernelBackupId: 789, KernelBackupName: "backup-20250115-1400", PdClusterId: 123, DfsPrefix: "backup-prefix", DfsBucket: "backup-bucket", DfsRegion: "us-west-2", ClusterVersion: "v7.5.0", SharedPoolId: 123, CloudProvider: "aws", }, }, 101: { Backup: &schema.Backup{ ClusterId: 101, TenantId: 1001, ServicePlan: "Premium", DisplayName: "auto-backup-202501151400", BackupTime: backupTime, BackupSizeBytes: 1024 * 1024 * 200, State: schema.BackupStateSuccess, Type: schema.BackupTypeSnapshot, TriggerType: schema.BackupTriggerTypeAuto, KernelBackupId: 790, KernelBackupName: "backup-20250115-1400", PdClusterId: 123, DfsPrefix: "backup-prefix", DfsBucket: "backup-bucket", DfsRegion: "us-west-2", ClusterVersion: "v7.5.0", SharedPoolId: 123, CloudProvider: "aws", }, }, } backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, TenantId: 1000, }, }, 101: { BackupSetting: &schema.BackupSetting{ ClusterId: 101, TenantId: 1001, }, }, } // Setup mock expectations mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }).Times(2) // Expect backup creation for cluster 100 mockBackupRepo.EXPECT().Create(ctx, records[100]).Return(nil) mockBackupSettingRepo.EXPECT().UpdateLastAutoBackupAt(ctx, uint64(100), &backupTime).Return(backupTime, nil) // Expect backup creation for cluster 101 mockBackupRepo.EXPECT().Create(ctx, records[101]).Return(nil) mockBackupSettingRepo.EXPECT().UpdateLastAutoBackupAt(ctx, uint64(101), &backupTime).Return(backupTime, nil) // Create backupSyncSlots instance s := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: backupSettings, } // Call the function errs := s.saveBackupSyncResult(ctx, records) // Assertions require.Empty(t, errs) assert.Equal(t, &backupTime, backupSettings[100].LastAutoBackupAt) assert.Equal(t, &backupTime, backupSettings[101].LastAutoBackupAt) }) t.Run("backup creation fails - should return error", func(t *testing.T) { // Setup mocks mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) // Create test data backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) records := map[uint64]*entity.Backup{ 100: { Backup: &schema.Backup{ ClusterId: 100, TenantId: 1000, BackupTime: backupTime, KernelBackupId: 789, }, }, } backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, TenantId: 1000, }, }, } // Setup mock expectations mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }) // Expect backup creation to fail expectedErr := bizerr.New(bizerr.ErrCodeUnknown, "database error") mockBackupRepo.EXPECT().Create(ctx, records[100]).Return(expectedErr) // Create backupSyncSlots instance s := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: backupSettings, } // Call the function errs := s.saveBackupSyncResult(ctx, records) // Assertions require.Error(t, errs[100]) assert.Contains(t, errs[100].Error(), "create backup record failed") assert.Nil(t, backupSettings[100].LastAutoBackupAt) // Should not be updated }) t.Run("backup setting not found - should return error", func(t *testing.T) { // Setup mocks mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) // Create test data backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) records := map[uint64]*entity.Backup{ 100: { Backup: &schema.Backup{ ClusterId: 100, TenantId: 1000, BackupTime: backupTime, KernelBackupId: 789, }, }, } // Empty backup settings - cluster 100 not found backupSettings := map[uint64]*entity.BackupSetting{} // Setup mock expectations mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }) // Expect backup creation to succeed mockBackupRepo.EXPECT().Create(ctx, records[100]).Return(nil) // Create backupSyncSlots instance s := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: backupSettings, } // Call the function errs := s.saveBackupSyncResult(ctx, records) // Assertions require.Error(t, errs[100]) assert.Contains(t, errs[100].Error(), "backup setting not found") }) t.Run("update last auto backup at fails - should return error", func(t *testing.T) { // Setup mocks mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) // Create test data backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) records := map[uint64]*entity.Backup{ 100: { Backup: &schema.Backup{ ClusterId: 100, TenantId: 1000, BackupTime: backupTime, KernelBackupId: 789, }, }, } backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, TenantId: 1000, }, }, } // Setup mock expectations mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }) // Expect backup creation to succeed mockBackupRepo.EXPECT().Create(ctx, records[100]).Return(nil) // Expect update to fail expectedErr := bizerr.New(bizerr.ErrCodeUnknown, "update failed") mockBackupSettingRepo.EXPECT().UpdateLastAutoBackupAt(ctx, uint64(100), &backupTime).Return(time.Time{}, expectedErr) // Create backupSyncSlots instance s := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: backupSettings, } // Call the function errs := s.saveBackupSyncResult(ctx, records) // Assertions require.Error(t, errs[100]) assert.Contains(t, errs[100].Error(), "update LastAutoBackupAt in database failed") assert.Nil(t, backupSettings[100].LastAutoBackupAt) // Should not be updated }) t.Run("deleted backup setting is evicted without save error", func(t *testing.T) { mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) record := &entity.Backup{Backup: &schema.Backup{ ClusterId: 100, SharedPoolId: 123, BackupTime: backupTime, }} setting := &entity.BackupSetting{BackupSetting: &schema.BackupSetting{ ClusterId: 100, SharedPoolId: 123, }} mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }) mockBackupRepo.EXPECT().Create(ctx, record).Return(nil) mockBackupSettingRepo.EXPECT(). UpdateLastAutoBackupAt(ctx, uint64(100), &backupTime). Return(time.Time{}, bizerr.New(bizerr.ErrCodeRecordNotFound, "backup setting not found")) slots := make([]map[uint64]map[uint64]bool, MinuteCountOfADay) slots[60] = map[uint64]map[uint64]bool{123: {100: true}} s := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: map[uint64]*entity.BackupSetting{100: setting}, clusterSlots: map[uint64][]int{100: {60}}, slots: slots, } records := map[uint64]*entity.Backup{100: record} require.Empty(t, s.saveBackupSyncResult(ctx, records)) assert.NotContains(t, records, uint64(100)) assert.NotContains(t, s.backupSettings, uint64(100)) assert.NotContains(t, s.clusterSlots, uint64(100)) assert.NotContains(t, s.slots[60][123], uint64(100)) }) t.Run("transaction fails - should return error", func(t *testing.T) { // Setup mocks mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) // Create test data backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) records := map[uint64]*entity.Backup{ 100: { Backup: &schema.Backup{ ClusterId: 100, TenantId: 1000, BackupTime: backupTime, KernelBackupId: 789, }, }, } backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, TenantId: 1000, }, }, } // Setup mock expectations - transaction callback succeeds but commit fails. expectedErr := bizerr.New(bizerr.ErrCodeUnknown, "transaction failed") mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { require.NoError(t, fn(ctx)) return expectedErr }) mockBackupRepo.EXPECT().Create(ctx, records[100]).Return(nil) mockBackupSettingRepo.EXPECT(). UpdateLastAutoBackupAt(ctx, uint64(100), &backupTime). Return(backupTime, nil) // Create backupSyncSlots instance s := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: backupSettings, } // Call the function errs := s.saveBackupSyncResult(ctx, records) // Assertions require.Error(t, errs[100]) assert.Contains(t, errs[100].Error(), "transaction failed") assert.Nil(t, backupSettings[100].LastAutoBackupAt) // Should not be updated }) t.Run("empty records - should succeed", func(t *testing.T) { // Setup mocks mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) // Empty records records := map[uint64]*entity.Backup{} backupSettings := map[uint64]*entity.BackupSetting{} // Create backupSyncSlots instance s := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: backupSettings, } // Call the function errs := s.saveBackupSyncResult(ctx, records) // Assertions require.Empty(t, errs) // No mock expectations needed since no records to process }) t.Run("partial failure - should not stop on first error", func(t *testing.T) { // Setup mocks mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) // Create test data with 2 records backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) records := map[uint64]*entity.Backup{ 100: { Backup: &schema.Backup{ ClusterId: 100, TenantId: 1000, BackupTime: backupTime, KernelBackupId: 789, }, }, 101: { Backup: &schema.Backup{ ClusterId: 101, TenantId: 1001, BackupTime: backupTime, KernelBackupId: 790, }, }, } backupSettings := map[uint64]*entity.BackupSetting{ 100: { BackupSetting: &schema.BackupSetting{ ClusterId: 100, TenantId: 1000, }, }, 101: { BackupSetting: &schema.BackupSetting{ ClusterId: 101, TenantId: 1001, }, }, } // Setup mock expectations - first transaction succeeds, second fails mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }) // First record fails during backup creation expectedErr := bizerr.New(bizerr.ErrCodeUnknown, "database error") mockBackupRepo.EXPECT().Create(ctx, records[100]).Return(expectedErr) // Second transaction fails mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }) // Second record succeeds mockBackupRepo.EXPECT().Create(ctx, records[101]).Return(nil) mockBackupSettingRepo.EXPECT().UpdateLastAutoBackupAt(ctx, uint64(101), &backupTime).Return(backupTime, nil) // Create backupSyncSlots instance s := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: backupSettings, } // Call the function errs := s.saveBackupSyncResult(ctx, records) // Assertions require.Len(t, errs, 1) require.Error(t, errs[100]) assert.Contains(t, errs[100].Error(), "create backup record failed") // First record should be updated in memory assert.Equal(t, &backupTime, backupSettings[101].LastAutoBackupAt) // Second record should not be updated assert.Nil(t, backupSettings[100].LastAutoBackupAt) }) t.Run("nil backup settings - should handle gracefully", func(t *testing.T) { // Setup mocks mockBackupRepo := mockbrrepo.NewMockBackupRepo(ctrl) mockBackupSettingRepo := mockbrrepo.NewMockBackupSettingRepo(ctrl) mockTransaction := externalmocks.NewMockTransactioner(ctrl) // Create test data backupTime := time.Date(2025, 1, 15, 14, 0, 0, 0, time.UTC) records := map[uint64]*entity.Backup{ 100: { Backup: &schema.Backup{ ClusterId: 100, TenantId: 1000, BackupTime: backupTime, KernelBackupId: 789, }, }, } // Nil backup settings var backupSettings map[uint64]*entity.BackupSetting // Setup mock expectations mockTransaction.EXPECT().Transaction(ctx, gomock.Any()).DoAndReturn( func(ctx context.Context, fn func(context.Context) error) error { return fn(ctx) }) // Expect backup creation to succeed mockBackupRepo.EXPECT().Create(ctx, records[100]).Return(nil) // Create backupSyncSlots instance s := &backupSyncSlots{ backupRepo: mockBackupRepo, backupSettingRepo: mockBackupSettingRepo, transaction: mockTransaction, backupSettings: backupSettings, } // Call the function errs := s.saveBackupSyncResult(ctx, records) // Assertions require.Error(t, errs[100]) assert.Contains(t, errs[100].Error(), "backup setting not found") }) } func deepClone[T any](src T) (T, error) { var buf bytes.Buffer err := gob.NewEncoder(&buf).Encode(src) if err != nil { var zero T return zero, err } var dst T err = gob.NewDecoder(&buf).Decode(&dst) return dst, err } func TestGetBackupExpirationTime(t *testing.T) { t.Run("empty backup policy uses RetentionDays", func(t *testing.T) { backupTime := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC) backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ BackupPolicy: "", Schedule: schema.AutoSchedule{ RetentionDays: 14, }, }, } result := getBackupExpirationTime(backupTime, backupTime, backupSetting) expected := backupTime.Add(14 * 24 * time.Hour) assert.Equal(t, expected, result) }) t.Run("backup policy set and backup at UTC midnight uses ZeroClockRetentionDays", func(t *testing.T) { backupTime := time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC) backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ BackupPolicy: schema.BackupPolicyNameTIER1, Schedule: schema.AutoSchedule{ Type: "per-hour", RetentionDays: 14, ZeroClockRetentionDays: 30, NonZeroClockRetentionDays: 7, }, }, } result := getBackupExpirationTime(backupTime, backupTime, backupSetting) expected := backupTime.Add(30 * 24 * time.Hour) assert.Equal(t, expected, result) }) t.Run("backup policy set and backup NOT at UTC midnight uses NonZeroClockRetentionDays", func(t *testing.T) { backupTime := time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC) backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ BackupPolicy: schema.BackupPolicyNameTIER1, Schedule: schema.AutoSchedule{ Type: "per-hour", RetentionDays: 14, ZeroClockRetentionDays: 30, NonZeroClockRetentionDays: 7, }, }, } result := getBackupExpirationTime(backupTime, backupTime, backupSetting) expected := backupTime.Add(7 * 24 * time.Hour) assert.Equal(t, expected, result) }) t.Run("backup policy set and backup at non-UTC midnight uses NonZeroClockRetentionDays", func(t *testing.T) { // 8:00 AM in UTC+8 is 00:00 AM in UTC, but we pass non-UTC time // The function should convert to UTC first loc := time.FixedZone("UTC+8", 8*60*60) backupTime := time.Date(2024, 1, 15, 8, 0, 0, 0, loc) // This is 00:00 UTC backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ BackupPolicy: schema.BackupPolicyNameTIER1, Schedule: schema.AutoSchedule{ Type: "per-hour", ZeroClockRetentionDays: 30, NonZeroClockRetentionDays: 7, }, }, } result := getBackupExpirationTime(backupTime, backupTime, backupSetting) // Since 8:00 UTC+8 = 00:00 UTC, it should use ZeroClockRetentionDays expected := backupTime.Add(30 * 24 * time.Hour) assert.Equal(t, expected, result) }) t.Run("backup at UTC midnight with nanoseconds uses ZeroClockRetentionDays", func(t *testing.T) { // Not exactly midnight due to nanoseconds backupTime := time.Date(2024, 1, 15, 0, 0, 0, 1, time.UTC) backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ BackupPolicy: schema.BackupPolicyNameTIER2, Schedule: schema.AutoSchedule{ Type: "per-hour", ZeroClockRetentionDays: 30, NonZeroClockRetentionDays: 7, }, }, } result := getBackupExpirationTime(backupTime, backupTime, backupSetting) // Not exactly midnight, so should use ZeroClockRetentionDays expected := backupTime.Add(30 * 24 * time.Hour) assert.Equal(t, expected, result) }) t.Run("backup policy set uses slot time for daily retention and actual time as base", func(t *testing.T) { backupTime := time.Date(2024, 1, 15, 0, 1, 15, 0, time.UTC) slotTime := time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC) backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ BackupPolicy: schema.BackupPolicyNameTIER1, Schedule: schema.AutoSchedule{ Type: "per-hour", DailyBackupMinuteOfDay: 0, ZeroClockRetentionDays: 30, NonZeroClockRetentionDays: 7, }, }, } result := getBackupExpirationTime(backupTime, slotTime, backupSetting) expected := backupTime.Add(30 * 24 * time.Hour) assert.Equal(t, expected, result) }) t.Run("flexible backup uses daily retention", func(t *testing.T) { backupTime := time.Date(2024, 1, 15, 10, 1, 15, 0, time.UTC) slotTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC) backupSetting := &entity.BackupSetting{ BackupSetting: &schema.BackupSetting{ BackupPolicy: schema.BackupPolicyNameTIER1, Mode: schema.BackupModeFlexibleRetention, Schedule: schema.AutoSchedule{ Type: "per-hour", DailyBackupMinuteOfDay: 0, ZeroClockRetentionDays: 30, NonZeroClockRetentionDays: 7, }, }, } result := getBackupExpirationTime(backupTime, slotTime, backupSetting) expected := backupTime.Add(30 * 24 * time.Hour) assert.Equal(t, expected, result) }) }