diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/product.go b/pkg/microservice/aslan/core/common/repository/mongodb/product.go index 4608143b22..757c06a5f5 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/product.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/product.go @@ -110,6 +110,13 @@ func (c *ProductColl) GetCollectionName() string { func (c *ProductColl) EnsureIndex(ctx context.Context) error { mod := []mongo.IndexModel{ + { + Keys: bson.D{ + bson.E{Key: "product_name", Value: 1}, + bson.E{Key: "env_name", Value: 1}, + }, + Options: options.Index().SetUnique(true), + }, { Keys: bson.D{ bson.E{Key: "env_name", Value: 1}, diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/production_service.go b/pkg/microservice/aslan/core/common/repository/mongodb/production_service.go index d6667ba3da..8688eae615 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/production_service.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/production_service.go @@ -409,42 +409,7 @@ func (c *ProductionServiceColl) ListMaxRevisionsByProductWithFilter(productName } func (c *ProductionServiceColl) ListServicesWithSRevision(opt *SvcRevisionListOption) ([]*models.Service, error) { - productMatch := bson.M{} - productMatch["product_name"] = opt.ProductName - - var serviceMatch bson.A - for _, sr := range opt.ServiceRevisions { - serviceMatch = append(serviceMatch, bson.M{ - "service_name": sr.ServiceName, - "revision": sr.Revision, - }) - } - - pipeline := []bson.M{ - { - "$match": productMatch, - }, - } - if len(opt.ServiceRevisions) > 0 { - pipeline = append(pipeline, bson.M{ - "$match": bson.M{ - "$or": serviceMatch, - }, - }) - } else { - return []*models.Service{}, nil - } - - cursor, err := c.Aggregate(context.TODO(), pipeline) - if err != nil { - return nil, err - } - - res := make([]*models.Service, 0) - if err := cursor.All(context.TODO(), &res); err != nil { - return nil, err - } - return res, err + return listServicesWithSRevision(c.Collection, opt) } func (c *ProductionServiceColl) TransferServiceSource(productName, serviceName, source, newSource, username, yaml string) error { diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/service.go b/pkg/microservice/aslan/core/common/repository/mongodb/service.go index 97cb59e4e2..b71a666e79 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/service.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/service.go @@ -575,42 +575,55 @@ func (c *ServiceColl) ListAllRevisions() ([]*models.Service, error) { } func (c *ServiceColl) ListServicesWithSRevision(opt *SvcRevisionListOption) ([]*models.Service, error) { - productMatch := bson.M{} - productMatch["product_name"] = opt.ProductName - - var serviceMatch bson.A - for _, sr := range opt.ServiceRevisions { - serviceMatch = append(serviceMatch, bson.M{ - "service_name": sr.ServiceName, - "revision": sr.Revision, - }) - } + return listServicesWithSRevision(c.Collection, opt) +} - pipeline := []bson.M{ - { - "$match": productMatch, - }, - } - if len(opt.ServiceRevisions) > 0 { - pipeline = append(pipeline, bson.M{ - "$match": bson.M{ - "$or": serviceMatch, - }, - }) - } else { +func listServicesWithSRevision(collection *mongo.Collection, opt *SvcRevisionListOption) ([]*models.Service, error) { + filter, ok := serviceRevisionListFilter(opt) + if !ok { return []*models.Service{}, nil } - cursor, err := c.Aggregate(context.TODO(), pipeline) + cursor, err := collection.Find(context.TODO(), filter) if err != nil { return nil, err } + defer cursor.Close(context.TODO()) res := make([]*models.Service, 0) if err := cursor.All(context.TODO(), &res); err != nil { return nil, err } - return res, err + return res, nil +} + +func serviceRevisionListFilter(opt *SvcRevisionListOption) (bson.M, bool) { + if len(opt.ServiceRevisions) == 0 { + return nil, false + } + revisionsByService := make(map[string]map[int64]struct{}) + for _, sr := range opt.ServiceRevisions { + if revisionsByService[sr.ServiceName] == nil { + revisionsByService[sr.ServiceName] = make(map[int64]struct{}) + } + revisionsByService[sr.ServiceName][sr.Revision] = struct{}{} + } + + serviceMatch := make(bson.A, 0, len(revisionsByService)) + for serviceName, revisionSet := range revisionsByService { + revisions := make([]int64, 0, len(revisionSet)) + for revision := range revisionSet { + revisions = append(revisions, revision) + } + serviceMatch = append(serviceMatch, bson.M{ + "service_name": serviceName, + "revision": bson.M{"$in": revisions}, + }) + } + return bson.M{ + "product_name": opt.ProductName, + "$or": serviceMatch, + }, true } func (c *ServiceColl) ListMaxRevisionsByProject(serviceName, serviceType string) ([]*models.Service, error) { diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/service_module.go b/pkg/microservice/aslan/core/common/repository/mongodb/service_module.go index 90cf8cfd98..5c5acc1ffd 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/service_module.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/service_module.go @@ -143,6 +143,38 @@ func (c *ServiceModuleColl) ListByServiceRevision(ctx context.Context, projectNa })) } +// ListByServiceRevisions returns all manual records in a project together +// with the visible auto-discovered records for the requested service +// revisions. It is the batch equivalent of ListByServiceRevision and keeps +// the same deterministic ordering required by the merge logic. +func (c *ServiceModuleColl) ListByServiceRevisions(ctx context.Context, projectName string, serviceRevisions map[string][]int64) ([]*models.ServiceModule, error) { + autoMatches := make(bson.A, 0, len(serviceRevisions)) + for serviceName, revisions := range serviceRevisions { + if serviceName == "" || len(revisions) == 0 { + continue + } + autoMatches = append(autoMatches, bson.M{ + "service_name": serviceName, + "revision_bound": bson.M{"$in": revisions}, + }) + } + + moduleMatches := bson.A{bson.M{"is_manual": true}} + if len(autoMatches) > 0 { + moduleMatches = append(moduleMatches, bson.M{ + "is_manual": false, + "ignored": bson.M{"$ne": true}, + "$or": autoMatches, + }) + } + + query := bson.M{ + "project_name": projectName, + "$or": moduleMatches, + } + return c.findAll(ctx, query, options.Find().SetSort(bson.D{{Key: "create_time", Value: 1}, {Key: "_id", Value: 1}})) +} + // ListManual returns every manual record for a service. Used by the manual- // module CRUD API to list user-declared modules independently of any revision. func (c *ServiceModuleColl) ListManual(ctx context.Context, projectName, serviceName string) ([]*models.ServiceModule, error) { diff --git a/pkg/microservice/aslan/core/common/service/repository/service_module.go b/pkg/microservice/aslan/core/common/service/repository/service_module.go index 31d215e7f0..8d6e7cdab9 100644 --- a/pkg/microservice/aslan/core/common/service/repository/service_module.go +++ b/pkg/microservice/aslan/core/common/service/repository/service_module.go @@ -37,6 +37,63 @@ type ModuleConflict struct { Shadowed []*models.ServiceModule } +// ServiceModuleSnapshot holds modules preloaded for one request. +type ServiceModuleSnapshot struct { + Manual map[string][]*models.Container + Resolved map[string]map[int64][]*models.Container + RecordCount int +} + +// LoadServiceModuleSnapshot loads all records required for serviceRevisions +// in one MongoDB find and prepares the same merge results as +// ResolveServiceModules. +func LoadServiceModuleSnapshot(ctx context.Context, projectName string, production bool, serviceRevisions map[string][]int64) (*ServiceModuleSnapshot, error) { + records, err := pickServiceModuleColl(production).ListByServiceRevisions(ctx, projectName, serviceRevisions) + if err != nil { + return nil, err + } + return newServiceModuleSnapshot(records, serviceRevisions), nil +} + +func newServiceModuleSnapshot(records []*models.ServiceModule, serviceRevisions map[string][]int64) *ServiceModuleSnapshot { + snapshot := &ServiceModuleSnapshot{ + Manual: make(map[string][]*models.Container), + Resolved: make(map[string]map[int64][]*models.Container), + RecordCount: len(records), + } + recordsByService := make(map[string][]*models.ServiceModule) + for _, record := range records { + if record == nil { + continue + } + recordsByService[record.ServiceName] = append(recordsByService[record.ServiceName], record) + if record.IsManual { + snapshot.Manual[record.ServiceName] = append(snapshot.Manual[record.ServiceName], &models.Container{ + Name: record.Name, Type: record.Type, Image: record.Image, ImageName: record.ImageName, ImagePath: record.ImagePath, + }) + } + } + + for serviceName, revisions := range serviceRevisions { + snapshot.Resolved[serviceName] = make(map[int64][]*models.Container) + for _, revision := range revisions { + selected := make([]*models.ServiceModule, 0) + for _, record := range recordsByService[serviceName] { + // ListByServiceRevision historically excludes ignored records for + // both branches, while the standalone manual listing does not. + if record.Ignored { + continue + } + if record.IsManual || record.RevisionBound == revision { + selected = append(selected, record) + } + } + snapshot.Resolved[serviceName][revision] = mergeServiceModules(selected) + } + } + return snapshot +} + // ResolveServiceModules returns the merged module list for one (project, // service, revision) plus any name conflicts. Production picks the right // underlying collection (service_module vs production_service_module). diff --git a/pkg/microservice/aslan/core/common/service/service.go b/pkg/microservice/aslan/core/common/service/service.go index e69331a1f7..bc06155343 100644 --- a/pkg/microservice/aslan/core/common/service/service.go +++ b/pkg/microservice/aslan/core/common/service/service.go @@ -1100,26 +1100,42 @@ func ListServicesInEnv(envName, productName string, newSvcKVsMap map[string][]*c return BuildServiceInfoInEnv(env, latestSvcs, newSvcKVsMap, log) } +// BuildServiceInfoOptions supplies data already loaded by a request-scoped caller. +type BuildServiceInfoOptions struct { + Project *template.Product + ProductTemplateServices []*commonmodels.Service + Modules *repository.ServiceModuleSnapshot +} + // @fixme newSvcKVsMap is old struct kv map, which are the kv are from deploy job config // may need to be removed, or use new kv struct // helm values need to be refactored -func BuildServiceInfoInEnv(productInfo *commonmodels.Product, templateSvcs []*commonmodels.Service, newSvcKVsMap map[string][]*commonmodels.ServiceKeyVal, log *zap.SugaredLogger) (*EnvServices, error) { +func BuildServiceInfoInEnv(productInfo *commonmodels.Product, templateSvcs []*commonmodels.Service, newSvcKVsMap map[string][]*commonmodels.ServiceKeyVal, log *zap.SugaredLogger, preload ...*BuildServiceInfoOptions) (*EnvServices, error) { productName, envName := productInfo.ProductName, productInfo.EnvName ret := &EnvServices{ ProductName: productName, EnvName: envName, Services: make([]*EnvService, 0), } - - project, err := templaterepo.NewProductColl().Find(productInfo.ProductName) - if err != nil { - return nil, e.ErrGetService.AddDesc(fmt.Sprintf("failed to find project %s, err: %v", productInfo.ProductName, err)) + var project *template.Product + var productTemplateSvcs []*commonmodels.Service + var moduleSnapshot *repository.ServiceModuleSnapshot + var err error + if len(preload) > 0 && preload[0] != nil { + project = preload[0].Project + productTemplateSvcs = preload[0].ProductTemplateServices + moduleSnapshot = preload[0].Modules + } else { + project, err = templaterepo.NewProductColl().Find(productInfo.ProductName) + if err != nil { + return nil, e.ErrGetService.AddDesc(fmt.Sprintf("failed to find project %s, err: %v", productName, err)) + } + productTemplateSvcs, err = commonutil.GetProductUsedTemplateSvcs(productInfo) + if err != nil { + return nil, e.ErrGetService.AddErr(errors.Wrapf(err, "failed to find product template services for env %s:%s", productName, envName)) + } } - productTemplateSvcs, err := commonutil.GetProductUsedTemplateSvcs(productInfo) - if err != nil { - return nil, e.ErrGetService.AddErr(errors.Wrapf(err, "failed to find product template services for env %s:%s", productName, envName)) - } productTemplateSvcMap := make(map[string]*commonmodels.Service) for _, svc := range productTemplateSvcs { productTemplateSvcMap[svc.ServiceName] = svc @@ -1131,11 +1147,15 @@ func BuildServiceInfoInEnv(productInfo *commonmodels.Product, templateSvcs []*co templateSvcMap[svc.ServiceName] = svc svcModulesMap[svc.ServiceName] = make(map[string]*commonmodels.Container) - // Service.Containers is no longer persisted — read modules from the - // service_module collection. - resolved, _, rerr := repository.ResolveServiceModules(context.Background(), svc.ProductName, svc.ServiceName, productInfo.Production, svc.Revision) - if rerr != nil { - return nil, e.ErrGetService.AddErr(errors.Wrapf(rerr, "failed to resolve modules for %s/%s rev %d", svc.ProductName, svc.ServiceName, svc.Revision)) + var resolved []*commonmodels.Container + if moduleSnapshot != nil { + resolved = moduleSnapshot.Resolved[svc.ServiceName][svc.Revision] + } else { + var rerr error + resolved, _, rerr = repository.ResolveServiceModules(context.Background(), svc.ProductName, svc.ServiceName, productInfo.Production, svc.Revision) + if rerr != nil { + return nil, e.ErrGetService.AddErr(errors.Wrapf(rerr, "failed to resolve modules for %s/%s rev %d", svc.ProductName, svc.ServiceName, svc.Revision)) + } } for _, container := range resolved { svcModulesMap[svc.ServiceName][container.Name] = container @@ -1157,8 +1177,9 @@ func BuildServiceInfoInEnv(productInfo *commonmodels.Product, templateSvcs []*co ret := make([]*commonmodels.Container, 0) if modulesMap, ok := svcModulesMap[svcName]; ok { for _, module := range modulesMap { - module.ImageName = commonutil.ExtractImageName(module.Image) - ret = append(ret, module) + copy := *module + copy.ImageName = commonutil.ExtractImageName(module.Image) + ret = append(ret, ©) } } return ret diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go index d9cc47b00c..e5317c98b4 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/controller/job/job_deploy.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "strings" + "time" "golang.org/x/exp/slices" @@ -46,7 +47,116 @@ import ( type DeployJobController struct { *BasicInfo - jobSpec *commonmodels.ZadigDeployJobSpec + jobSpec *commonmodels.ZadigDeployJobSpec + presetContext *deployPresetContext +} + +type deployPresetServiceRevision struct { + serviceName string + revision int64 +} + +// deployPresetContext caches Mongo data for one preset request. +type deployPresetContext struct { + loaded bool + environments []*commonmodels.Product + environmentByName map[string]*commonmodels.Product + projectInfo *templatemodels.Product + latestServices []*commonmodels.Service + serviceByRevision map[deployPresetServiceRevision]*commonmodels.Service + serviceModules *repository.ServiceModuleSnapshot + defaultRegistryID string +} + +func (c *deployPresetContext) loadProjectInfo(project string) error { + if c.projectInfo != nil { + return nil + } + projectInfo, err := templaterepo.NewProductColl().Find(project) + if err != nil { + return fmt.Errorf("failed to find project %s: %w", project, err) + } + c.projectInfo = projectInfo + return nil +} + +func (c *deployPresetContext) load(project string, production bool, envSource config.ParamSourceType, currentEnv string, ticket *commonmodels.ApprovalTicket) error { + if c.loaded { + return nil + } + + started := time.Now() + environments, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{Name: project, Production: util.GetBoolPointer(production)}) + if err != nil { + return fmt.Errorf("failed to list environments for project %s: %w", project, err) + } + + shouldLoadEnv := func(environment *commonmodels.Product) bool { + if !ticket.IsAllowedEnv(project, environment.EnvName) { + return false + } + return envSource != config.ParamSourceFixed || environment.EnvName == currentEnv + } + + environmentByName := make(map[string]*commonmodels.Product, len(environments)) + requestedRevisions := make(map[deployPresetServiceRevision]struct{}) + for _, environment := range environments { + environment.LintServices() + environmentByName[environment.EnvName] = environment + if !shouldLoadEnv(environment) { + continue + } + for _, service := range environment.GetServiceMap() { + requestedRevisions[deployPresetServiceRevision{service.ServiceName, service.Revision}] = struct{}{} + } + } + + if err := c.loadProjectInfo(project); err != nil { + return err + } + latestServices, err := repository.ListMaxRevisionsServices(project, production, false) + if err != nil { + return fmt.Errorf("failed to list latest services for project %s: %w", project, err) + } + revisionOptions := make([]*commonrepo.ServiceRevision, 0, len(requestedRevisions)) + for revision := range requestedRevisions { + revisionOptions = append(revisionOptions, &commonrepo.ServiceRevision{ServiceName: revision.serviceName, Revision: revision.revision}) + } + environmentServices, err := repository.ListServicesWithSRevision(&commonrepo.SvcRevisionListOption{ProductName: project, ServiceRevisions: revisionOptions}, production) + if err != nil { + return fmt.Errorf("failed to list environment service revisions for project %s: %w", project, err) + } + serviceByRevision := make(map[deployPresetServiceRevision]*commonmodels.Service, len(environmentServices)) + for _, service := range environmentServices { + serviceByRevision[deployPresetServiceRevision{service.ServiceName, service.Revision}] = service + } + moduleRevisions := make(map[string][]int64, len(latestServices)) + for _, service := range latestServices { + moduleRevisions[service.ServiceName] = []int64{service.Revision} + } + serviceModules, err := repository.LoadServiceModuleSnapshot(context.Background(), project, production, moduleRevisions) + if err != nil { + return fmt.Errorf("failed to load service modules for project %s: %w", project, err) + } + + registry, err := commonrepo.NewRegistryNamespaceColl().Find(&commonrepo.FindRegOps{IsDefault: true}) + if err != nil { + return fmt.Errorf("failed to find default registry for project %s: %w", project, err) + } + + c.loaded, c.environments, c.environmentByName = true, environments, environmentByName + c.latestServices, c.serviceByRevision = latestServices, serviceByRevision + c.serviceModules, c.defaultRegistryID = serviceModules, registry.ID.Hex() + log.Debugf("loaded deploy preset context for project %s in %s: environments=%d service_revisions=%d module_records=%d", + project, time.Since(started), len(environments), len(requestedRevisions), serviceModules.RecordCount) + return nil +} + +func (j DeployJobController) loadPresetContext(ticket *commonmodels.ApprovalTicket) (*deployPresetContext, error) { + if err := j.presetContext.load(j.workflow.Project, j.jobSpec.Production, j.jobSpec.EnvSource, j.jobSpec.Env, ticket); err != nil { + return nil, err + } + return j.presetContext, nil } func CreateDeployJobController(job *commonmodels.Job, workflow *commonmodels.WorkflowV4) (Job, error) { @@ -64,8 +174,9 @@ func CreateDeployJobController(job *commonmodels.Job, workflow *commonmodels.Wor } return DeployJobController{ - BasicInfo: basicInfo, - jobSpec: spec, + BasicInfo: basicInfo, + jobSpec: spec, + presetContext: &deployPresetContext{}, }, nil } @@ -134,11 +245,11 @@ func (j DeployJobController) Update(useUserInput bool, ticket *commonmodels.Appr j.executePolicy = latestJob.ExecutePolicy j.jobSpec.Production = latestSpec.Production - project, err := templaterepo.NewProductColl().Find(j.workflow.Project) - if err != nil { - return fmt.Errorf("failed to find project %s, err: %v", j.workflow.Project, err) + if err := j.presetContext.loadProjectInfo(j.workflow.Project); err != nil { + return err } + project := j.presetContext.projectInfo if project.ProductFeature != nil { j.jobSpec.DeployType = project.ProductFeature.DeployType } @@ -175,31 +286,24 @@ func (j DeployJobController) Update(useUserInput bool, ticket *commonmodels.Appr if !ticket.IsAllowedEnv(j.workflow.Project, j.jobSpec.Env) { j.jobSpec.Env = "" } - products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ - Name: j.workflow.Project, - Production: util.GetBoolPointer(j.jobSpec.Production), - }) + + // if unselected for some reason, we skip calculating default service + if j.jobSpec.Env == "" { + j.jobSpec.Services = make([]*commonmodels.DeployServiceInfo, 0) + return nil + } + presetContext, err := j.loadPresetContext(ticket) if err != nil { - log.Errorf("can't list envs in project %s, error: %w", j.workflow.Project, err) return err } - - currentEnvMap := make(map[string]*commonmodels.Product) - for _, env := range products { - currentEnvMap[env.EnvName] = env - } - - if _, ok := currentEnvMap[j.jobSpec.Env]; !ok { + environment, ok := presetContext.environmentByName[j.jobSpec.Env] + if !ok { j.jobSpec.Env = "" - } - - // if unselected for some reason, we skip calculating default service - if j.jobSpec.Env == "" { j.jobSpec.Services = make([]*commonmodels.DeployServiceInfo, 0) return nil } - envDeployInfo, err := generateDeployInfoForEnv(j.jobSpec.Env, j.workflow.Project, j.jobSpec.Production, j.jobSpec.ServiceVariableConfig, ticket) + envDeployInfo, err := generateDeployInfoForEnvironment(environment, presetContext, j.jobSpec.ServiceVariableConfig, ticket) if err != nil { log.Errorf("failed to generate service deployment info for env: %s, error: %s", j.jobSpec.Env, err) return err @@ -231,7 +335,17 @@ func (j DeployJobController) Update(useUserInput bool, ticket *commonmodels.Appr return nil } - envDeployInfo, err := generateDeployInfoForEnv(j.jobSpec.Env, j.workflow.Project, j.jobSpec.Production, j.jobSpec.ServiceVariableConfig, ticket) + presetContext, err := j.loadPresetContext(ticket) + if err != nil { + return err + } + var envDeployInfo *commonmodels.ZadigDeployEnvInformation + environment, ok := presetContext.environmentByName[j.jobSpec.Env] + if !ok { + err = fmt.Errorf("failed to find env: %s in environments", j.jobSpec.Env) + } else { + envDeployInfo, err = generateDeployInfoForEnvironment(environment, presetContext, j.jobSpec.ServiceVariableConfig, ticket) + } if err != nil { log.Errorf("failed to generate service deployment info for env: %s, error: %s", j.jobSpec.Env, err) if latestSpec.EnvSource == config.ParamSourceFixed { @@ -303,28 +417,30 @@ func (j DeployJobController) Update(useUserInput bool, ticket *commonmodels.Appr // it will update the env's option/ service's option based on the user's setting on whether they update the config func (j DeployJobController) SetOptions(ticket *commonmodels.ApprovalTicket) error { envOptions := make([]*commonmodels.ZadigDeployEnvInformation, 0) + if j.jobSpec.EnvSource == config.ParamSourceFixed && !ticket.IsAllowedEnv(j.workflow.Project, j.jobSpec.Env) { + j.jobSpec.EnvOptions = envOptions + return nil + } + presetContext, err := j.loadPresetContext(ticket) + if err != nil { + return err + } + generationStarted := time.Now() if j.jobSpec.EnvSource == config.ParamSourceFixed { - if ticket.IsAllowedEnv(j.workflow.Project, j.jobSpec.Env) { - envInfo, err := generateDeployInfoForEnv(j.jobSpec.Env, j.workflow.Project, j.jobSpec.Production, j.jobSpec.ServiceVariableConfig, ticket) - if err != nil { - log.Errorf("failed to generate service deployment info for env: %s, error: %s", j.jobSpec.Env, err) - return err - } - - envOptions = append(envOptions, envInfo) + environment, ok := presetContext.environmentByName[j.jobSpec.Env] + if !ok { + return fmt.Errorf("failed to find env: %s in environments", j.jobSpec.Env) } - } else { - products, err := commonrepo.NewProductColl().List(&commonrepo.ProductListOptions{ - Name: j.workflow.Project, - Production: util.GetBoolPointer(j.jobSpec.Production), - }) + envInfo, err := generateDeployInfoForEnvironment(environment, presetContext, j.jobSpec.ServiceVariableConfig, ticket) if err != nil { - log.Errorf("can't list envs in project %s, error: %w", j.workflow.Project, err) + log.Errorf("failed to generate service deployment info for env: %s, error: %s", j.jobSpec.Env, err) return err } - for _, env := range products { + envOptions = append(envOptions, envInfo) + } else { + for _, env := range presetContext.environments { // skip the sleeping envs if env.IsSleeping() { continue @@ -334,7 +450,7 @@ func (j DeployJobController) SetOptions(ticket *commonmodels.ApprovalTicket) err continue } - envInfo, err := generateDeployInfoForEnv(env.EnvName, j.workflow.Project, j.jobSpec.Production, j.jobSpec.ServiceVariableConfig, ticket) + envInfo, err := generateDeployInfoForEnvironment(env, presetContext, j.jobSpec.ServiceVariableConfig, ticket) if err != nil { log.Errorf("failed to generate service deployment info for env: %s, error: %s", j.jobSpec.Env, err) return err @@ -404,6 +520,7 @@ func (j DeployJobController) SetOptions(ticket *commonmodels.ApprovalTicket) err } j.jobSpec.EnvOptions = envOptions + log.Debugf("generated deploy preset options for project %s in %s: environments=%d", j.workflow.Project, time.Since(generationStarted), len(envOptions)) return nil } @@ -957,38 +1074,22 @@ ServiceOrderLoop: return resp, nil } -// generateDeployInfoForEnv generates the whole environment deployment info for the given env, it contains: +// generateDeployInfoForEnvironment generates deployment information from a +// request-scoped, preloaded data set. It contains: // 1. basic environment information // 2. ALL service information that is either in the environment or the service definition // 3. 2 versions of variable list (filtered by the user's variable configuration) showing the service variable definition and the env variables (if applicable) -func generateDeployInfoForEnv(env, project string, production bool, configuredServiceVariableList commonmodels.DeployServiceVariableConfigList, approvalTicket *commonmodels.ApprovalTicket) (*commonmodels.ZadigDeployEnvInformation, error) { - repositoryCache := repository.NewRepsitoryCache() - serviceOption := make([]*commonmodels.DeployOptionInfo, 0) - - envInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ - Name: project, - EnvName: env, - Production: util.GetBoolPointer(production), - }) - if err != nil { - return nil, fmt.Errorf("failed to find env: %s in environments, error: %s", env, err) +func generateDeployInfoForEnvironment(envInfo *commonmodels.Product, presetContext *deployPresetContext, configuredServiceVariableList commonmodels.DeployServiceVariableConfigList, approvalTicket *commonmodels.ApprovalTicket) (*commonmodels.ZadigDeployEnvInformation, error) { + env := envInfo.EnvName + project := envInfo.ProductName + production := envInfo.Production + registryID := envInfo.RegistryID + if registryID == "" { + registryID = presetContext.defaultRegistryID } + serviceOption := make([]*commonmodels.DeployOptionInfo, 0) - if envInfo.RegistryID == "" { - registry, err := commonrepo.NewRegistryNamespaceColl().Find(&commonrepo.FindRegOps{ - IsDefault: true, - }) - - if err != nil { - return nil, fmt.Errorf("failed to find default registry for env: %s, error: %s", env, err) - } - envInfo.RegistryID = registry.ID.Hex() - } - - projectInfo, err := templaterepo.NewProductColl().Find(project) - if err != nil { - return nil, fmt.Errorf("failed to find project %s, err: %v", project, err) - } + projectInfo := presetContext.projectInfo envServiceMap := envInfo.GetServiceMap() @@ -1008,10 +1109,7 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe } // Phase 4: append manual modules for host-product env case. - manualMods, mErr := repository.ListManualServiceModules(context.Background(), project, service.ServiceName, production) - if mErr != nil { - return nil, fmt.Errorf("failed to list manual service modules for %s: %v", service.ServiceName, mErr) - } + manualMods := presetContext.serviceModules.Manual[service.ServiceName] for _, m := range manualMods { if _, exists := modulesMap[m.Name]; exists { continue @@ -1046,36 +1144,27 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe Env: envInfo.EnvName, EnvAlias: envInfo.Alias, Production: production, - RegistryID: envInfo.RegistryID, + RegistryID: registryID, Services: serviceOption, }, nil } serviceDefinitionMap := make(map[string]*commonmodels.Service) - var serviceDefinitions []*commonmodels.Service - if production { - serviceDefinitions, err = commonrepo.NewProductionServiceColl().ListMaxRevisions(&commonrepo.ServiceListOption{ - ProductName: project, - }) - } else { - serviceDefinitions, err = commonrepo.NewServiceColl().ListMaxRevisions(&commonrepo.ServiceListOption{ - ProductName: project, - }) - } - if err != nil { - return nil, fmt.Errorf("failed to list services, error: %s", err) - } + serviceDefinitions := presetContext.latestServices for _, service := range serviceDefinitions { serviceDefinitionMap[service.ServiceName] = service } - serviceList, err := repository.ListMaxRevisionsServices(project, production, false) - if err != nil { - return nil, fmt.Errorf("get service definition list error: %v", err) + productTemplateServices := make([]*commonmodels.Service, 0, len(envServiceMap)) + for _, service := range envServiceMap { + if definition := presetContext.serviceByRevision[deployPresetServiceRevision{serviceName: service.ServiceName, revision: service.Revision}]; definition != nil { + productTemplateServices = append(productTemplateServices, definition) + } } - - serviceGeneralInfo, err := commonservice.BuildServiceInfoInEnv(envInfo, serviceList, nil, log.SugaredLogger()) + serviceGeneralInfo, err := commonservice.BuildServiceInfoInEnv(envInfo, serviceDefinitions, nil, log.SugaredLogger(), &commonservice.BuildServiceInfoOptions{ + Project: projectInfo, ProductTemplateServices: productTemplateServices, Modules: presetContext.serviceModules, + }) if err != nil { return nil, fmt.Errorf("failed to generate service info, error: %s", err) } @@ -1111,10 +1200,7 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe // for CRD/DaemonSet workloads) and dedup against modulesMap so we // don't double-count when a manual record shares a name with a // container already pulled from ProductService via the env reconcile. - manualMods, mErr := repository.ListManualServiceModules(context.Background(), project, service.ServiceName, production) - if mErr != nil { - return nil, fmt.Errorf("failed to list manual service modules for %s: %v", service.ServiceName, mErr) - } + manualMods := presetContext.serviceModules.Manual[service.ServiceName] for _, m := range manualMods { if _, exists := modulesMap[m.Name]; exists { continue @@ -1131,13 +1217,9 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe currentReleaseName := "" if service.Type == setting.HelmDeployType { - envService, err := repositoryCache.QueryTemplateServiceWithCache(&commonrepo.ServiceFindOption{ - ProductName: project, - ServiceName: service.ServiceName, - Revision: service.Revision, - }, production) - if err != nil { - return nil, fmt.Errorf("failed to query template service %s/%s/%d, error: %s", project, service.ServiceName, service.Revision, err) + envService, ok := presetContext.serviceByRevision[deployPresetServiceRevision{serviceName: service.ServiceName, revision: service.Revision}] + if !ok { + return nil, fmt.Errorf("failed to query template service %s/%s/%d", project, service.ServiceName, service.Revision) } currentReleaseName = util.GeneReleaseName(envService.GetReleaseNaming(), project, envInfo.Namespace, env, service.ServiceName) @@ -1147,10 +1229,7 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe if serviceDef, ok := serviceDefinitionMap[service.ServiceName]; ok { // Service.Containers no longer persisted — pull merged modules // for the latest template revision. - defContainers, _, dErr := repository.ResolveServiceModules(context.Background(), serviceDef.ProductName, serviceDef.ServiceName, production, serviceDef.Revision) - if dErr != nil { - return nil, fmt.Errorf("failed to resolve modules for %s/%s rev %d: %s", serviceDef.ProductName, serviceDef.ServiceName, serviceDef.Revision, dErr) - } + defContainers := presetContext.serviceModules.Resolved[serviceDef.ServiceName][serviceDef.Revision] for _, module := range defContainers { // if a container is newly created in the service, add it to the module list if _, ok := modulesMap[module.Name]; !ok { @@ -1222,10 +1301,7 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe // Service.Containers is no longer persisted; read modules from the // normalized module store so services not deployed in the env still // expose their selectable modules in workflow presets. - containers, _, rErr := repository.ResolveServiceModules(context.Background(), service.ProductName, service.ServiceName, production, service.Revision) - if rErr != nil { - return nil, fmt.Errorf("failed to resolve modules for %s/%s rev %d: %s", service.ProductName, service.ServiceName, service.Revision, rErr) - } + containers := presetContext.serviceModules.Resolved[service.ServiceName][service.Revision] for _, module := range containers { modulesMap[module.Name] = struct{}{} if approvalTicket.IsAllowedService(project, service.ServiceName, module.Name) { @@ -1238,10 +1314,7 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe } // Phase 4: append manual modules for services not yet deployed in env. - manualMods, mErr := repository.ListManualServiceModules(context.Background(), project, service.ServiceName, production) - if mErr != nil { - return nil, fmt.Errorf("failed to list manual service modules for %s: %v", service.ServiceName, mErr) - } + manualMods := presetContext.serviceModules.Manual[service.ServiceName] for _, m := range manualMods { if _, exists := modulesMap[m.Name]; exists { continue @@ -1299,7 +1372,7 @@ func generateDeployInfoForEnv(env, project string, production bool, configuredSe Env: envInfo.EnvName, EnvAlias: envInfo.Alias, Production: production, - RegistryID: envInfo.RegistryID, + RegistryID: registryID, Services: serviceOption, }, nil }