Skip to content

Commit 80b4eea

Browse files
JigarJoshiJigarJoshi
committed
fix: Break the tenant reload transaction further
1 parent b1a26b1 commit 80b4eea

File tree

6 files changed

+57
-41
lines changed

6 files changed

+57
-41
lines changed

‎server/metadata/tenant.go‎

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -188,12 +188,10 @@ func (m *TenantManager) CreateOrGetTenant(ctx context.Context, namespace Namespa
188188

189189
defer func() {
190190
if err == nil {
191-
if err = tx.Commit(ctx); err == nil {
192-
// commit succeed, so we can safely cache it now, for other workers it may happen as part of the
193-
// first call in query lifecycle
194-
m.tenants[namespace.StrId()] = tenant
195-
m.idToTenantMap[namespace.Id()] = namespace.StrId()
196-
}
191+
// commit succeed, so we can safely cache it now, for other workers it may happen as part of the
192+
// first call in query lifecycle
193+
m.tenants[namespace.StrId()] = tenant
194+
m.idToTenantMap[namespace.Id()] = namespace.StrId()
197195
} else {
198196
_ = tx.Rollback(ctx)
199197
}
@@ -379,7 +377,7 @@ func (m *TenantManager) GetTenant(ctx context.Context, namespaceId string) (*Ten
379377
namespace := NewTenantNamespace(namespaceId, metadata)
380378
tenant = NewTenant(namespace, m.kvStore, m.searchStore,
381379
m.metaStore, m.encoder, m.versionH, currentVersion, m.tableKeyGenerator)
382-
if err = tenant.reload(ctx, tx, currentVersion, m.searchSchemasSnapshot); err != nil {
380+
if err = tenant.reload(ctx, tx, currentVersion, m.searchSchemasSnapshot, m.txMgr); err != nil {
383381
return nil, err
384382
}
385383

@@ -464,7 +462,7 @@ func (m *TenantManager) createOrGetTenantInternal(ctx context.Context, tx transa
464462

465463
tenant := NewTenant(namespace, m.kvStore, m.searchStore, m.metaStore, m.encoder, m.versionH, currentVersion, m.tableKeyGenerator)
466464
tenant.Lock()
467-
err = tenant.reload(ctx, tx, currentVersion, m.searchSchemasSnapshot)
465+
err = tenant.reload(ctx, tx, currentVersion, m.searchSchemasSnapshot, m.txMgr)
468466
tenant.Unlock()
469467
return tenant, err
470468
}
@@ -584,17 +582,13 @@ func (m *TenantManager) reload(ctx context.Context, currentVersion Version) erro
584582
return err
585583
}
586584
tenant.Lock()
587-
err = tenant.reload(ctx, tx, currentVersion, m.searchSchemasSnapshot)
585+
err = tenant.reload(ctx, tx, currentVersion, m.searchSchemasSnapshot, m.txMgr)
588586
tenant.Unlock()
589587
if err != nil {
590588
log.Err(err).Msgf("reloading a tenant failed '%s'", tenant.name)
591589
_ = tx.Rollback(ctx)
592590
return err
593591
}
594-
if err = tx.Commit(ctx); err != nil {
595-
log.Err(err).Msgf("committing a reloading of tenant failed '%s'", tenant.name)
596-
return err
597-
}
598592
}
599593
return nil
600594
}
@@ -650,7 +644,7 @@ func NewTenant(namespace Namespace, kvStore kv.TxStore, searchStore search.Store
650644
// thread will actually perform reload. This is a blocking API which means if most of the requests detected that the
651645
// tenant state is stale then they all will block till one of them will reload the tenant state from the database. All
652646
// the blocking transactions will be restarted to ensure they see the latest view of the tenant.
653-
func (tenant *Tenant) Reload(ctx context.Context, tx transaction.Tx, version Version, searchSchemasSnapshot map[string]*tsApi.CollectionResponse) error {
647+
func (tenant *Tenant) Reload(ctx context.Context, tx transaction.Tx, version Version, searchSchemasSnapshot map[string]*tsApi.CollectionResponse, txMgr *transaction.Manager) error {
654648
if !tenant.shouldReload(version) {
655649
return nil
656650
}
@@ -662,7 +656,7 @@ func (tenant *Tenant) Reload(ctx context.Context, tx transaction.Tx, version Ver
662656
return nil
663657
}
664658

665-
return tenant.reload(ctx, tx, version, searchSchemasSnapshot)
659+
return tenant.reload(ctx, tx, version, searchSchemasSnapshot, txMgr)
666660
}
667661

668662
func (tenant *Tenant) shouldReload(currentVersion Version) bool {
@@ -678,7 +672,7 @@ func (tenant *Tenant) shouldReload(currentVersion Version) bool {
678672
// loads all the databases, it loads the resources for each one. Once databases are reloaded then it performs the same
679673
// logic for search indexes. Once search indexes are loaded it links back the search indexes to the Tigris Collection
680674
// if the source for these search indexes is Tigris.
681-
func (tenant *Tenant) reload(ctx context.Context, tx transaction.Tx, currentVersion Version, searchSchemasSnapshot map[string]*tsApi.CollectionResponse) error {
675+
func (tenant *Tenant) reload(ctx context.Context, tx transaction.Tx, currentVersion Version, searchSchemasSnapshot map[string]*tsApi.CollectionResponse, txMgr *transaction.Manager) error {
682676
// reset
683677
tenant.projects = make(map[string]*Project)
684678
tenant.idToDatabaseMap = make(map[uint32]*Database)
@@ -714,11 +708,14 @@ func (tenant *Tenant) reload(ctx context.Context, tx transaction.Tx, currentVers
714708
tenant.idToDatabaseMap[meta.ID] = database
715709
}
716710

711+
err = tx.Commit(ctx)
712+
if err != nil {
713+
log.Fatal().Err(err).Msg("Failed to reload tenant")
714+
}
717715
// load search indexes, this is essentially loading all the search indexes created by the user and attaching it to
718716
// the project object.
719717
for _, p := range tenant.projects {
720-
var err error
721-
if p.search, err = tenant.reloadSearch(ctx, tx, p, searchSchemasSnapshot); err != nil {
718+
if p.search, err = tenant.reloadSearch(ctx, txMgr, p, searchSchemasSnapshot); err != nil {
722719
return err
723720
}
724721
for _, index := range p.search.indexes {
@@ -805,16 +802,32 @@ func (tenant *Tenant) reloadDatabase(ctx context.Context, tx transaction.Tx, dbN
805802
}
806803

807804
// reloadSearch is responsible for reloading all the search indexes inside a single project.
808-
func (tenant *Tenant) reloadSearch(ctx context.Context, tx transaction.Tx, project *Project, searchSchemasSnapshot map[string]*tsApi.CollectionResponse) (*Search, error) {
809-
projMetadata, err := tenant.namespaceStore.GetProjectMetadata(ctx, tx, tenant.namespace.Id(), project.Name())
805+
func (tenant *Tenant) reloadSearch(ctx context.Context, txMgr *transaction.Manager, project *Project, searchSchemasSnapshot map[string]*tsApi.CollectionResponse) (*Search, error) {
806+
txToReadProjectMetadata, err := txMgr.StartTx(ctx)
807+
if err != nil {
808+
log.Fatal().Err(err).Msg("Failed to start tx to read project metadata while reloading tenant")
809+
}
810+
811+
projMetadata, err := tenant.namespaceStore.GetProjectMetadata(ctx, txToReadProjectMetadata, tenant.namespace.Id(), project.Name())
810812
if err != nil {
811813
return nil, errors.Internal("failed to get project metadata for project %s", project.Name())
812814
}
815+
_ = txToReadProjectMetadata.Commit(ctx)
813816

814817
searchObj := NewSearch()
815818

816-
for _, searchMD := range projMetadata.SearchMetadata {
817-
schV, err := tenant.searchSchemaStore.GetLatest(ctx, tx, tenant.namespace.Id(), project.id, searchMD.Name)
819+
var indexLevelTx transaction.Tx
820+
for i, searchMD := range projMetadata.SearchMetadata {
821+
if i%10 == 0 {
822+
if indexLevelTx != nil {
823+
_ = indexLevelTx.Commit(ctx)
824+
}
825+
indexLevelTx, err = txMgr.StartTx(ctx)
826+
if err != nil {
827+
log.Fatal().Err(err).Msg("Failed to start tx to reload indices in batch for tenant reload")
828+
}
829+
}
830+
schV, err := tenant.searchSchemaStore.GetLatest(ctx, indexLevelTx, tenant.namespace.Id(), project.id, searchMD.Name)
818831
if err != nil {
819832
return nil, err
820833
}
@@ -833,6 +846,9 @@ func (tenant *Tenant) reloadSearch(ctx context.Context, tx transaction.Tx, proje
833846
searchObj.indexes[searchMD.Name] = schema.NewSearchIndex(schV.Version, searchStoreIndexName, searchFactory, fieldsInSearchStore)
834847
}
835848

849+
if indexLevelTx != nil {
850+
_ = indexLevelTx.Commit(ctx)
851+
}
836852
return searchObj, nil
837853
}
838854

‎server/metadata/tenant_test.go‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ func TestTenantManager_CreateProjects(t *testing.T) {
224224

225225
tx, err = tm.StartTx(ctx)
226226
require.NoError(t, err)
227-
err = tenant.reload(ctx, tx, nil, nil)
227+
err = tenant.reload(ctx, tx, nil, nil, tm)
228228
require.NoError(t, err)
229229
proj1, err := tenant.GetProject(tenantProj1)
230230
require.NoError(t, err)
@@ -272,7 +272,7 @@ func TestTenantManager_DatabaseBranches(t *testing.T) {
272272
databases := (&Project{}).GetDatabaseWithBranches()
273273
require.Len(t, databases, 0)
274274

275-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
275+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
276276

277277
require.NoError(t, tenant.CreateBranch(ctx, tx, tenantProj1, NewDatabaseNameWithBranch(tenantProj1, "branch1")))
278278
require.NoError(t, tenant.CreateBranch(ctx, tx, tenantProj2, NewDatabaseNameWithBranch(tenantProj2, "branch1")))
@@ -282,7 +282,7 @@ func TestTenantManager_DatabaseBranches(t *testing.T) {
282282
require.ErrorContains(t, tenant.CreateBranch(ctx, tx, unknownProject, NewDatabaseNameWithBranch(unknownProject, "branch1")), "project doesn't exist")
283283

284284
// reload again to get all the branches
285-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
285+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
286286

287287
// list all branches
288288
branches := tenant.ListDatabaseBranches(tenantProj1)
@@ -339,7 +339,7 @@ func TestTenantManager_DatabaseBranches(t *testing.T) {
339339
require.NoError(t, tenant.DeleteBranch(ctx, tx, tenantProj1, NewDatabaseNameWithBranch(tenantProj1, "branch2")))
340340
require.ErrorContains(t, tenant.DeleteBranch(ctx, tx, unknownProject, NewDatabaseNameWithBranch(unknownProject, "branch1")), "project doesn't exist")
341341

342-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
342+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
343343
require.NoError(t, tx.Commit(ctx))
344344

345345
tx, err = tm.StartTx(ctx)
@@ -402,7 +402,7 @@ func TestTenantManager_CreateCollections(t *testing.T) {
402402
err = tenant.CreateProject(ctx, tx, tenantProj2, nil)
403403
require.NoError(t, err)
404404

405-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
405+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
406406

407407
proj1, err := tenant.GetProject(tenantProj1)
408408
require.NoError(t, err)
@@ -444,7 +444,7 @@ func TestTenantManager_CreateCollections(t *testing.T) {
444444
require.NoError(t, err)
445445
require.NoError(t, tenant.CreateCollection(ctx, tx, db2, factory))
446446

447-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
447+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
448448

449449
proj2, err = tenant.GetProject(tenantProj2)
450450
require.NoError(t, err)
@@ -477,7 +477,7 @@ func TestTenantManager_DropCollection(t *testing.T) {
477477
err = tenant.CreateProject(ctx, tx, tenantProj2, nil)
478478
require.NoError(t, err)
479479

480-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
480+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
481481

482482
proj1, err := tenant.GetProject(tenantProj1)
483483
require.NoError(t, err)
@@ -507,7 +507,7 @@ func TestTenantManager_DropCollection(t *testing.T) {
507507
factory, err := schema.NewFactoryBuilder(true).Build("test_collection", jsSchema)
508508
require.NoError(t, err)
509509
require.NoError(t, tenant.CreateCollection(ctx, tx, db2, factory))
510-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
510+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
511511
require.NoError(t, tx.Commit(ctx))
512512

513513
tx, err = tm.StartTx(ctx)
@@ -549,7 +549,7 @@ func TestTenantManager_SearchIndexes(t *testing.T) {
549549
err = tenant.CreateProject(ctx, tx, tenantProj1, nil)
550550
require.NoError(t, err)
551551

552-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
552+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
553553

554554
proj1, err := tenant.GetProject(tenantProj1)
555555
require.NoError(t, err)
@@ -578,7 +578,7 @@ func TestTenantManager_SearchIndexes(t *testing.T) {
578578
require.NoError(t, err)
579579
require.NotNil(t, indexesInSearchStore[tenant.Encoder.EncodeSearchTableName(tenant.namespace.Id(), proj1.Id(), factory.Name)])
580580

581-
require.NoError(t, tenant.reload(ctx, tx, nil, indexesInSearchStore))
581+
require.NoError(t, tenant.reload(ctx, tx, nil, indexesInSearchStore, tm))
582582

583583
proj1, err = tenant.GetProject(tenantProj1)
584584
require.NoError(t, err)
@@ -615,7 +615,7 @@ func TestTenantManager_SecondaryIndexes(t *testing.T) {
615615
err = tenant.CreateProject(ctx, tx, tenantProj2, nil)
616616
require.NoError(t, err)
617617

618-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
618+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
619619

620620
proj1, err := tenant.GetProject(tenantProj1)
621621
require.NoError(t, err)
@@ -657,7 +657,7 @@ func TestTenantManager_SecondaryIndexes(t *testing.T) {
657657
require.NoError(t, err)
658658
require.NoError(t, tenant.CreateCollection(ctx, tx, db2, factory))
659659

660-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
660+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
661661

662662
proj2, err = tenant.GetProject(tenantProj2)
663663
require.NoError(t, err)
@@ -688,7 +688,7 @@ func TestTenantManager_SecondaryIndexes(t *testing.T) {
688688
err = tenant.CreateProject(ctx, tx, tenantProj1, nil)
689689
require.NoError(t, err)
690690

691-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
691+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
692692

693693
proj1, err := tenant.GetProject(tenantProj1)
694694
require.NoError(t, err)
@@ -720,7 +720,7 @@ func TestTenantManager_SecondaryIndexes(t *testing.T) {
720720
require.NoError(t, err)
721721
require.NoError(t, tenant.CreateCollection(ctx, tx, db1, factory))
722722

723-
require.NoError(t, tenant.reload(ctx, tx, nil, nil))
723+
require.NoError(t, tenant.reload(ctx, tx, nil, nil, tm))
724724

725725
proj1, err = tenant.GetProject(tenantProj1)
726726
require.NoError(t, err)
@@ -1098,7 +1098,7 @@ func TestTenantManager_SearchDataSize(t *testing.T) {
10981098

10991099
err = tenant.CreateProject(ctx, tmTx, tenantProj2, nil)
11001100
require.NoError(t, err)
1101-
require.NoError(t, tenant.reload(ctx, tmTx, nil, nil))
1101+
require.NoError(t, tenant.reload(ctx, tmTx, nil, nil, tm))
11021102

11031103
// proj1
11041104
proj1, err := tenant.GetProject(tenantProj1)

‎server/metadata/tenant_tracker.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ func (cacheTracker *CacheTracker) stopTracking(ctx context.Context, tenant *Tena
202202
return err
203203
}
204204

205-
if err = tenant.Reload(ctx, tx, version, cacheTracker.tenantMgr.searchSchemasSnapshot); err != nil {
205+
if err = tenant.Reload(ctx, tx, version, cacheTracker.tenantMgr.searchSchemasSnapshot, cacheTracker.txMgr); err != nil {
206206
return err
207207
}
208208

‎server/quota/quota_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func TestQuota(t *testing.T) {
7070
factory, err := schema.NewFactoryBuilder(true).Build("test_collection", jsSchema)
7171
require.NoError(t, err)
7272

73-
err = tenant.Reload(ctx, tx, []byte("aaa"), nil)
73+
err = tenant.Reload(ctx, tx, []byte("aaa"), nil, txMgr)
7474
require.NoError(t, err)
7575

7676
proj1, err := tenant.GetProject(projName)

‎server/quota/storage_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ func TestStorageQuota(t *testing.T) {
6161
factory, err := schema.NewFactoryBuilder(true).Build("test_collection", jsSchema)
6262
require.NoError(t, err)
6363

64-
err = tenant.Reload(ctx, tx, []byte("aaa"), nil)
64+
err = tenant.Reload(ctx, tx, []byte("aaa"), nil, txMgr)
6565
require.NoError(t, err)
6666

6767
proj1, err := tenant.GetProject(projName)

‎server/services/v1/realtime/device_session.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func (s *Sessions) CreateDeviceSession(ctx context.Context, conn *websocket.Conn
7373
if version, err = s.versionH.Read(ctx, tx, false); err != nil {
7474
return nil, err
7575
}
76-
if err = tenant.Reload(ctx, tx, version, nil); err != nil {
76+
if err = tenant.Reload(ctx, tx, version, nil, s.txMgr); err != nil {
7777
return nil, err
7878
}
7979

0 commit comments

Comments
 (0)