From 54039e7e80ebc181ec6d45419e2868d9353d223d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Gryglicki?= Date: Wed, 19 Aug 2026 05:22:20 +0000 Subject: [PATCH 1/7] Search CLA group API - 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) Assisted by [Claude](https://claude.ai) --- cla-backend-go/cmd/server.go | 3 + cla-backend-go/swagger/cla.v2.yaml | 58 ++ .../swagger/common/cla-search-list.yaml | 26 + .../swagger/common/cla-search-org.yaml | 18 + .../swagger/common/cla-search-result.yaml | 53 ++ cla-backend-go/v2/cla_search/handlers.go | 63 +++ cla-backend-go/v2/cla_search/handlers_test.go | 75 +++ cla-backend-go/v2/cla_search/repository.go | 326 +++++++++++ cla-backend-go/v2/cla_search/service.go | 505 ++++++++++++++++++ cla-backend-go/v2/cla_search/service_test.go | 451 ++++++++++++++++ utils/cla_search.sh | 95 ++++ 11 files changed, 1673 insertions(+) create mode 100644 cla-backend-go/swagger/common/cla-search-list.yaml create mode 100644 cla-backend-go/swagger/common/cla-search-org.yaml create mode 100644 cla-backend-go/swagger/common/cla-search-result.yaml create mode 100644 cla-backend-go/v2/cla_search/handlers.go create mode 100644 cla-backend-go/v2/cla_search/handlers_test.go create mode 100644 cla-backend-go/v2/cla_search/repository.go create mode 100644 cla-backend-go/v2/cla_search/service.go create mode 100644 cla-backend-go/v2/cla_search/service_test.go create mode 100755 utils/cla_search.sh diff --git a/cla-backend-go/cmd/server.go b/cla-backend-go/cmd/server.go index bdfb3e1cc..2c4b58e2e 100644 --- a/cla-backend-go/cmd/server.go +++ b/cla-backend-go/cmd/server.go @@ -103,6 +103,7 @@ import ( "github.com/linuxfoundation/easycla/cla-backend-go/template" "github.com/linuxfoundation/easycla/cla-backend-go/user" v2ClaManager "github.com/linuxfoundation/easycla/cla-backend-go/v2/cla_manager" + v2ClaSearch "github.com/linuxfoundation/easycla/cla-backend-go/v2/cla_search" v2Company "github.com/linuxfoundation/easycla/cla-backend-go/v2/company" v2CurrentUser "github.com/linuxfoundation/easycla/cla-backend-go/v2/current_user" v2Health "github.com/linuxfoundation/easycla/cla-backend-go/v2/health" @@ -438,6 +439,7 @@ func server(localMode bool) http.Handler { gitlabOrganizationsService := gitlab_organizations.NewService(gitlabOrganizationRepo, v2RepositoriesService, v1ProjectClaGroupRepo, storeRepository, usersService, signaturesRepo, v1CompanyRepo) v1SignaturesService := signatures.NewService(signaturesRepo, v1CompanyService, usersService, eventsService, githubOrgValidation, v1RepositoriesService, githubOrganizationsService, v1ProjectService, gitlabApp, configFile.ClaV1ApiURL, configFile.CLALandingPage, configFile.CLALogoURL) v2SignatureService := v2Signatures.NewService(awsSession, configFile.SignatureFilesBucket, v1ProjectService, v1CompanyService, v1SignaturesService, v1ProjectClaGroupRepo, signaturesRepo, usersService, approvalsRepo) + v2ClaSearchService := v2ClaSearch.NewService(v2ClaSearch.NewRepository(awsSession, stage)) v2MyClasService := v2MyClas.NewService(v2MyClas.NewRepository(awsSession, stage), user_service.GetClient(), v1SignaturesService, v1CompanyRepo, v1ProjectClaGroupRepo, project_service.GetClient()) v1ClaManagerService := cla_manager.NewService(claManagerReqRepo, v1ProjectClaGroupRepo, v1CompanyService, v1ProjectService, usersService, v1SignaturesService, eventsService, emailTemplateService, configFile.CorporateConsoleV2URL) v2ClaManagerService := v2ClaManager.NewService(emailTemplateService, v1CompanyService, v1ProjectService, v1ClaManagerService, usersService, v1RepositoriesService, v2CompanyService, eventsService, v1ProjectClaGroupRepo) @@ -514,6 +516,7 @@ func server(localMode bool) http.Handler { v2Company.Configure(v2API, v2CompanyService, v1ProjectClaGroupRepo, configFile.LFXPortalURL) v2CurrentUser.Configure(v2API, v2CurrentUserService) v2MyClas.Configure(v2API, v2MyClasService) + v2ClaSearch.Configure(v2API, v2ClaSearchService) cla_manager.Configure(api, v1ClaManagerService, v1CompanyService, v1ProjectService, usersService, v1SignaturesService, eventsService, emailTemplateService) v2ClaManager.Configure(v2API, v2ClaManagerService, v1CompanyService, configFile.LFXPortalURL, configFile.CorporateConsoleV2URL, v1ProjectClaGroupRepo, userRepo) cla_groups.Configure(v2API, v2ClaGroupService, v1ProjectService, v1ProjectClaGroupRepo, eventsService) diff --git a/cla-backend-go/swagger/cla.v2.yaml b/cla-backend-go/swagger/cla.v2.yaml index c2c35bd41..eec9d0ab7 100644 --- a/cla-backend-go/swagger/cla.v2.yaml +++ b/cla-backend-go/swagger/cla.v2.yaml @@ -2862,6 +2862,38 @@ paths: tags: - my_clas + /cla-group/search: + get: + summary: Search CLA Groups + description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit + operationId: searchClaGroups + parameters: + - $ref: "#/parameters/x-request-id" + - $ref: "#/parameters/x-acl" + - $ref: "#/parameters/x-username" + - $ref: "#/parameters/x-email" + - $ref: "#/parameters/claSearchTerm" + - $ref: "#/parameters/claSearchLimit" + responses: + '200': + description: 'Success' + headers: + x-request-id: + type: string + description: The unique request ID value - assigned/set by the API Gateway based on the session + schema: + $ref: '#/definitions/cla-search-list' + '400': + $ref: '#/responses/invalid-request' + '401': + $ref: '#/responses/unauthorized' + '403': + $ref: '#/responses/forbidden' + '500': + $ref: '#/responses/internal-server-error' + tags: + - cla_search + /user/{userID}/request-company-admin: post: summary: Request Manager @@ -5036,6 +5068,23 @@ parameters: type: string collectionFormat: multi required: false + claSearchTerm: + name: searchTerm + description: The term to search for - matched case-insensitively as a substring against the CLA Group name, project/foundation name and linked organization names, and resolved as a repository URL or "owner/repo" path + in: query + type: string + minLength: 3 + maxLength: 255 + required: true + claSearchLimit: + name: limit + description: The maximum number of CLA Groups to return + in: query + type: integer + minimum: 1 + maximum: 100 + default: 20 + required: false definitions: # Common definitions @@ -5067,6 +5116,15 @@ definitions: my-identity-list: $ref: './common/my-identity-list.yaml' + cla-search-list: + $ref: './common/cla-search-list.yaml' + + cla-search-result: + $ref: './common/cla-search-result.yaml' + + cla-search-org: + $ref: './common/cla-search-org.yaml' + #-------------------------------------- # Docusign Webhook Payload #____________________________________________ diff --git a/cla-backend-go/swagger/common/cla-search-list.yaml b/cla-backend-go/swagger/common/cla-search-list.yaml new file mode 100644 index 000000000..781c45296 --- /dev/null +++ b/cla-backend-go/swagger/common/cla-search-list.yaml @@ -0,0 +1,26 @@ +# Copyright The Linux Foundation and each contributor to CommunityBridge. +# SPDX-License-Identifier: MIT + +type: object +x-nullable: false +title: CLA Search List +description: The CLA Groups matching the search term +properties: + searchTerm: + type: string + description: The search term the results were resolved for + resultCount: + type: integer + format: int64 + x-omitempty: false + description: The number of results returned - at most limit + truncated: + type: boolean + x-omitempty: false + description: True when more CLA Groups matched than limit and the result set was capped - ask the user to refine the term + results: + type: array + x-omitempty: false + description: The matching CLA Groups, best match first + items: + $ref: '#/definitions/cla-search-result' diff --git a/cla-backend-go/swagger/common/cla-search-org.yaml b/cla-backend-go/swagger/common/cla-search-org.yaml new file mode 100644 index 000000000..f9af8f97a --- /dev/null +++ b/cla-backend-go/swagger/common/cla-search-org.yaml @@ -0,0 +1,18 @@ +# Copyright The Linux Foundation and each contributor to CommunityBridge. +# SPDX-License-Identifier: MIT + +type: object +x-nullable: false +title: CLA Search Organization +description: A repository-hosting organization (GitHub organization, GitLab group or Gerrit instance) linked to the CLA Group +properties: + name: + type: string + description: The organization/group/Gerrit instance name + source: + type: string + enum: [github, gitlab, gerrit] + description: The repository source hosting the organization + url: + type: string + description: The organization URL, omitted when the source record carries none diff --git a/cla-backend-go/swagger/common/cla-search-result.yaml b/cla-backend-go/swagger/common/cla-search-result.yaml new file mode 100644 index 000000000..535a5e374 --- /dev/null +++ b/cla-backend-go/swagger/common/cla-search-result.yaml @@ -0,0 +1,53 @@ +# Copyright The Linux Foundation and each contributor to CommunityBridge. +# SPDX-License-Identifier: MIT + +type: object +x-nullable: false +title: CLA Search Result +description: A CLA Group matching the search term - the CLA Group is the signing unit, so results are deduplicated by claGroupID +properties: + claGroupID: + type: string + description: The CLA Group ID (UUID) - the signing unit, used for the Contributor Console hand-off + claGroupName: + type: string + description: The CLA Group name, omitted when the CLA Group record could not be resolved + projectName: + type: string + description: The Salesforce project display name the CLA Group belongs to (a foundation-level CLA Group resolves to its foundation), omitted when the CLA Group maps to several projects with no foundation marker + projectSFID: + type: string + description: The Salesforce ID of the project in projectName, omitted when projectName could not be resolved + foundationSFID: + type: string + description: The Salesforce ID of the foundation the CLA Group belongs to, omitted when unknown + projectExternalID: + type: string + description: The external (Salesforce) ID stored on the CLA Group record, omitted when unset + iclaEnabled: + type: boolean + x-omitempty: false + description: The project_icla_enabled flag of the CLA Group + cclaEnabled: + type: boolean + x-omitempty: false + description: The project_ccla_enabled flag of the CLA Group + matchTypes: + type: array + x-omitempty: false + description: Why the CLA Group matched, sorted - claGroup (CLA Group name), project (project or foundation name), organization (linked organization name or URL), repository (the search term resolved to a repository) + items: + type: string + enum: [claGroup, project, organization, repository] + organizations: + type: array + x-omitempty: false + description: All repository-hosting organizations linked to the CLA Group, sorted by source then name - carries the repo-source provenance and backs the "N linked orgs" affordance + items: + $ref: '#/definitions/cla-search-org' + matchedRepositoryName: + type: string + description: The full repository name the search term resolved to, set only when matchTypes contains repository + matchedRepositoryURL: + type: string + description: The URL of the repository in matchedRepositoryName, set only when matchTypes contains repository diff --git a/cla-backend-go/v2/cla_search/handlers.go b/cla-backend-go/v2/cla_search/handlers.go new file mode 100644 index 000000000..3583562b8 --- /dev/null +++ b/cla-backend-go/v2/cla_search/handlers.go @@ -0,0 +1,63 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package cla_search + +import ( + "context" + "fmt" + "strings" + + "github.com/LF-Engineering/lfx-kit/auth" + "github.com/go-openapi/runtime/middleware" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/restapi/operations" + claSearchOps "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/restapi/operations/cla_search" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/sirupsen/logrus" +) + +const missingUsernameMsg = "the authenticated principal carries no username - unable to search" + +// authorized accepts a principal carrying a username, or an admin principal such as a machine token +func authorized(authUser *auth.User) bool { + return authUser != nil && (authUser.UserName != "" || utils.IsUserAdmin(authUser)) +} + +// Configure sets up the CLA Group search API handlers +func Configure(api *operations.EasyclaAPI, service Service) { + api.ClaSearchSearchClaGroupsHandler = claSearchOps.SearchClaGroupsHandlerFunc( + func(params claSearchOps.SearchClaGroupsParams, authUser *auth.User) middleware.Responder { + reqID := utils.GetRequestID(params.XREQUESTID) + ctx := context.WithValue(params.HTTPRequest.Context(), utils.XREQUESTID, reqID) // nolint + if authUser != nil { + utils.SetAuthUserProperties(authUser, params.XUSERNAME, params.XEMAIL) + } + f := logrus.Fields{ + "functionName": "v2.cla_search.handlers.SearchClaGroups", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "authUserName": utils.StringValue(params.XUSERNAME), + "searchTerm": params.SearchTerm, + } + + if !authorized(authUser) { + log.WithFields(f).Warn(missingUsernameMsg) + return claSearchOps.NewSearchClaGroupsUnauthorized().WithXRequestID(reqID).WithPayload(utils.ErrorResponseUnauthorized(reqID, missingUsernameMsg)) + } + + if len(strings.TrimSpace(params.SearchTerm)) < MinSearchTermLength { + msg := fmt.Sprintf("searchTerm must contain at least %d non-whitespace characters", MinSearchTermLength) + log.WithFields(f).Warn(msg) + return claSearchOps.NewSearchClaGroupsBadRequest().WithXRequestID(reqID).WithPayload(utils.ErrorResponseBadRequest(reqID, msg)) + } + + result, err := service.Search(ctx, params.SearchTerm, utils.Int64Value(params.Limit)) + if err != nil { + msg := "unable to search the CLA Groups for the provided search term" + log.WithFields(f).WithError(err).Warn(msg) + return claSearchOps.NewSearchClaGroupsInternalServerError().WithXRequestID(reqID).WithPayload(utils.ErrorResponseInternalServerErrorWithError(reqID, msg, err)) + } + + return claSearchOps.NewSearchClaGroupsOK().WithXRequestID(reqID).WithPayload(result) + }) +} diff --git a/cla-backend-go/v2/cla_search/handlers_test.go b/cla-backend-go/v2/cla_search/handlers_test.go new file mode 100644 index 000000000..8ccce798e --- /dev/null +++ b/cla-backend-go/v2/cla_search/handlers_test.go @@ -0,0 +1,75 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package cla_search + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/LF-Engineering/lfx-kit/auth" + "github.com/go-openapi/runtime" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/restapi/operations" + claSearchOps "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/restapi/operations/cla_search" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type stubService struct { + list *models.ClaSearchList + err error + term string +} + +func (s *stubService) Search(_ context.Context, searchTerm string, _ int64) (*models.ClaSearchList, error) { + s.term = searchTerm + return s.list, s.err +} + +func invoke(t *testing.T, svc Service, authUser *auth.User, searchTerm string) *httptest.ResponseRecorder { + t.Helper() + api := &operations.EasyclaAPI{} + Configure(api, svc) + require.NotNil(t, api.ClaSearchSearchClaGroupsHandler) + + params := claSearchOps.SearchClaGroupsParams{ + HTTPRequest: httptest.NewRequest(http.MethodGet, "/v4/cla-group/search", nil), + SearchTerm: searchTerm, + } + recorder := httptest.NewRecorder() + api.ClaSearchSearchClaGroupsHandler.Handle(params, authUser).WriteResponse(recorder, runtime.JSONProducer()) + return recorder +} + +func TestHandlerUnauthorizedWithoutPrincipal(t *testing.T) { + for name, authUser := range map[string]*auth.User{"nil": nil, "empty username": {}} { + t.Run(name, func(t *testing.T) { + svc := &stubService{} + assert.Equal(t, http.StatusUnauthorized, invoke(t, svc, authUser, "kubernetes").Code) + assert.Empty(t, svc.term) + }) + } +} + +func TestHandlerBadRequestOnWhitespaceOnlyTerm(t *testing.T) { + svc := &stubService{} + assert.Equal(t, http.StatusBadRequest, invoke(t, svc, &auth.User{UserName: "jdoe"}, " ").Code) + assert.Empty(t, svc.term) +} + +func TestHandlerInternalServerErrorOnServiceFailure(t *testing.T) { + svc := &stubService{err: errors.New("boom")} + assert.Equal(t, http.StatusInternalServerError, invoke(t, svc, &auth.User{UserName: "jdoe"}, "kubernetes").Code) +} + +func TestHandlerSuccess(t *testing.T) { + svc := &stubService{list: &models.ClaSearchList{SearchTerm: "kubernetes", ResultCount: 1, Results: []models.ClaSearchResult{{ClaGroupID: "cg-kube"}}}} + recorder := invoke(t, svc, &auth.User{UserName: "jdoe"}, "kubernetes") + assert.Equal(t, http.StatusOK, recorder.Code) + assert.Contains(t, recorder.Body.String(), `"claGroupID":"cg-kube"`) + assert.Equal(t, "kubernetes", svc.term) +} diff --git a/cla-backend-go/v2/cla_search/repository.go b/cla-backend-go/v2/cla_search/repository.go new file mode 100644 index 000000000..e60e8780f --- /dev/null +++ b/cla-backend-go/v2/cla_search/repository.go @@ -0,0 +1,326 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package cla_search + +import ( + "context" + "fmt" + "net/http" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" + "github.com/aws/aws-sdk-go/service/dynamodb/expression" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" +) + +// Scan segment counts, sized so every table is covered by one round of parallel 1MB scan pages - +// the CLA Group table carries the embedded CLA document bodies and is by far the largest +const ( + claGroupScanSegments = 8 + projectMappingScanSegments = 2 + orgScanSegments = 1 + + // the number of DynamoDB calls a single search can have in flight + searchConcurrency = claGroupScanSegments + projectMappingScanSegments + 3*orgScanSegments + 2 +) + +// ClaGroupRow is a CLA Group record, projected to the fields the search results carry +type ClaGroupRow struct { + ClaGroupID string `dynamodbav:"project_id"` + Name string `dynamodbav:"project_name"` + ExternalID string `dynamodbav:"project_external_id"` + IclaEnabled bool `dynamodbav:"project_icla_enabled"` + CclaEnabled bool `dynamodbav:"project_ccla_enabled"` +} + +// ProjectMappingRow is a projects-cla-groups mapping record, projected to the fields the search results carry +type ProjectMappingRow struct { + ClaGroupID string `dynamodbav:"cla_group_id"` + ClaGroupName string `dynamodbav:"cla_group_name"` + ProjectSFID string `dynamodbav:"project_sfid"` + ProjectName string `dynamodbav:"project_name"` + FoundationSFID string `dynamodbav:"foundation_sfid"` + FoundationName string `dynamodbav:"foundation_name"` +} + +// OrgRow is a repository-hosting organization - a GitHub organization, a GitLab group or a Gerrit instance +type OrgRow struct { + Name string + URL string + Source string + ProjectSFID string + ClaGroupID string +} + +// RepositoryRow is the repository a pasted URL or "owner/repo" path resolved to +type RepositoryRow struct { + Name string `dynamodbav:"repository_name"` + URL string `dynamodbav:"repository_url"` + Type string `dynamodbav:"repository_type"` + ClaGroupID string `dynamodbav:"repository_project_id"` +} + +// Repository interface defines the data access methods for the CLA Group search module +type Repository interface { + GetClaGroups(ctx context.Context) ([]*ClaGroupRow, error) + GetProjectMappings(ctx context.Context) ([]*ProjectMappingRow, error) + GetGithubOrgs(ctx context.Context) ([]*OrgRow, error) + GetGitlabOrgs(ctx context.Context) ([]*OrgRow, error) + GetGerritInstances(ctx context.Context) ([]*OrgRow, error) + GetRepositoriesByName(ctx context.Context, names []string) ([]*RepositoryRow, error) + GetRepositoriesByOrganization(ctx context.Context, organizationNames []string) ([]*RepositoryRow, error) +} + +type repository struct { + dynamoDBClient *dynamodb.DynamoDB + claGroupTableName string + projectMappingTableName string + githubOrgTableName string + gitlabOrgTableName string + gerritTableName string + repositoryTableName string +} + +// NewRepository creates a new instance of the CLA Group search repository +func NewRepository(awsSession *session.Session, stage string) Repository { + // a search fans out to more concurrent DynamoDB calls than the default two idle connections + // per host can serve, which would leave most of them paying for a fresh TLS handshake + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConns: searchConcurrency, + MaxIdleConnsPerHost: searchConcurrency, + IdleConnTimeout: 90 * time.Second, + } + + return repository{ + dynamoDBClient: dynamodb.New(awsSession.Copy(&aws.Config{HTTPClient: &http.Client{Transport: transport}})), + claGroupTableName: fmt.Sprintf("cla-%s-projects", stage), + projectMappingTableName: fmt.Sprintf("cla-%s-projects-cla-groups", stage), + githubOrgTableName: fmt.Sprintf("cla-%s-github-orgs", stage), + gitlabOrgTableName: fmt.Sprintf("cla-%s-gitlab-orgs", stage), + gerritTableName: fmt.Sprintf("cla-%s-gerrit-instances", stage), + repositoryTableName: fmt.Sprintf("cla-%s-repositories", stage), + } +} + +// enabledFilter keeps the disabled records - organizations unlinked from EasyCLA and repositories +// no longer covered by a CLA Group - out of the search, matching the convention of the +// github_organizations repository +func enabledFilter() expression.ConditionBuilder { + return expression.Name("enabled").Equal(expression.Value(true)) +} + +func (repo repository) GetClaGroups(ctx context.Context) ([]*ClaGroupRow, error) { + var rows []*ClaGroupRow + err := repo.scan(ctx, repo.claGroupTableName, claGroupScanSegments, nil, + []string{"project_id", "project_name", "project_external_id", "project_icla_enabled", "project_ccla_enabled"}, &rows) + return rows, err +} + +func (repo repository) GetProjectMappings(ctx context.Context) ([]*ProjectMappingRow, error) { + var rows []*ProjectMappingRow + err := repo.scan(ctx, repo.projectMappingTableName, projectMappingScanSegments, nil, + []string{"cla_group_id", "cla_group_name", "project_sfid", "project_name", "foundation_sfid", "foundation_name"}, &rows) + return rows, err +} + +type orgDBRow struct { + Name string `dynamodbav:"organization_name"` + URL string `dynamodbav:"organization_url"` + ProjectSFID string `dynamodbav:"project_sfid"` +} + +func (repo repository) GetGithubOrgs(ctx context.Context) ([]*OrgRow, error) { + return repo.scanOrgs(ctx, repo.githubOrgTableName, sourceGitHub) +} + +func (repo repository) GetGitlabOrgs(ctx context.Context) ([]*OrgRow, error) { + return repo.scanOrgs(ctx, repo.gitlabOrgTableName, sourceGitLab) +} + +func (repo repository) scanOrgs(ctx context.Context, tableName, source string) ([]*OrgRow, error) { + var rows []*orgDBRow + filter := enabledFilter() + if err := repo.scan(ctx, tableName, orgScanSegments, &filter, []string{"organization_name", "organization_url", "project_sfid"}, &rows); err != nil { + return nil, err + } + orgs := make([]*OrgRow, 0, len(rows)) + for _, row := range rows { + orgs = append(orgs, &OrgRow{Name: row.Name, URL: row.URL, Source: source, ProjectSFID: row.ProjectSFID}) + } + return orgs, nil +} + +type gerritDBRow struct { + Name string `dynamodbav:"gerrit_name"` + URL string `dynamodbav:"gerrit_url"` + ClaGroupID string `dynamodbav:"project_id"` +} + +func (repo repository) GetGerritInstances(ctx context.Context) ([]*OrgRow, error) { + var rows []*gerritDBRow + // the gerrit-instances table carries no enabled flag + if err := repo.scan(ctx, repo.gerritTableName, orgScanSegments, nil, []string{"gerrit_name", "gerrit_url", "project_id"}, &rows); err != nil { + return nil, err + } + orgs := make([]*OrgRow, 0, len(rows)) + for _, row := range rows { + orgs = append(orgs, &OrgRow{Name: row.Name, URL: row.URL, Source: sourceGerrit, ClaGroupID: row.ClaGroupID}) + } + return orgs, nil +} + +// GetRepositoriesByName resolves the given full repository names through the repository-name-index +// GSI - an exact hash-key lookup per name, no scan +func (repo repository) GetRepositoriesByName(ctx context.Context, names []string) ([]*RepositoryRow, error) { + return repo.queryRepositories(ctx, repositories.RepositoryNameIndex, "repository_name", names) +} + +// GetRepositoriesByOrganization returns the repositories of the given organizations through the +// repository-organization-name-index GSI, which is how a repository whose stored name is not +// lower-cased is recovered from a lower-cased pasted URL +func (repo repository) GetRepositoriesByOrganization(ctx context.Context, organizationNames []string) ([]*RepositoryRow, error) { + return repo.queryRepositories(ctx, repositories.RepositoryOrganizationNameIndex, "repository_organization_name", organizationNames) +} + +// queryRepositories runs one enabled-only GSI query per key value, in parallel +func (repo repository) queryRepositories(ctx context.Context, indexName, keyAttribute string, values []string) ([]*RepositoryRow, error) { + f := logrus.Fields{ + "functionName": "v2.cla_search.repository.queryRepositories", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "indexName": indexName, + "values": values, + } + + var ( + mu sync.Mutex + rows []*RepositoryRow + ) + group, groupCtx := errgroup.WithContext(ctx) + for _, value := range values { + keyValue := value + group.Go(func() error { + expr, err := expression.NewBuilder(). + WithKeyCondition(expression.Key(keyAttribute).Equal(expression.Value(keyValue))). + WithFilter(enabledFilter()). + WithProjection(expression.NamesList(expression.Name("repository_name"), expression.Name("repository_url"), + expression.Name("repository_type"), expression.Name("repository_project_id"))). + Build() + if err != nil { + log.WithFields(f).WithError(err).Warn("error building expression for the repository query") + return err + } + + queryInput := &dynamodb.QueryInput{ + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + FilterExpression: expr.Filter(), + KeyConditionExpression: expr.KeyCondition(), + ProjectionExpression: expr.Projection(), + TableName: aws.String(repo.repositoryTableName), + IndexName: aws.String(indexName), + } + for { + results, queryErr := repo.dynamoDBClient.QueryWithContext(groupCtx, queryInput) + if queryErr != nil { + log.WithFields(f).WithError(queryErr).Warn("error querying repositories") + return queryErr + } + + var page []*RepositoryRow + if unmarshalErr := dynamodbattribute.UnmarshalListOfMaps(results.Items, &page); unmarshalErr != nil { + log.WithFields(f).WithError(unmarshalErr).Warn("error unmarshalling repositories") + return unmarshalErr + } + + mu.Lock() + rows = append(rows, page...) + mu.Unlock() + + if len(results.LastEvaluatedKey) == 0 { + return nil + } + queryInput.ExclusiveStartKey = results.LastEvaluatedKey + } + }) + } + if err := group.Wait(); err != nil { + return nil, err + } + + return rows, nil +} + +// scan runs a projected full-table scan into out, which must be a pointer to a slice. The table is +// split into segments scanned in parallel so a table spanning several 1MB scan pages costs one +// round trip rather than one per page. +func (repo repository) scan(ctx context.Context, tableName string, segments int, filter *expression.ConditionBuilder, attributes []string, out interface{}) error { + f := logrus.Fields{ + "functionName": "v2.cla_search.repository.scan", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "tableName": tableName, + "segments": segments, + } + + names := make([]expression.NameBuilder, 0, len(attributes)) + for _, attribute := range attributes { + names = append(names, expression.Name(attribute)) + } + builder := expression.NewBuilder().WithProjection(expression.NamesList(names[0], names[1:]...)) + if filter != nil { + builder = builder.WithFilter(*filter) + } + expr, err := builder.Build() + if err != nil { + log.WithFields(f).WithError(err).Warn("error building expression for the scan") + return err + } + + var ( + mu sync.Mutex + items []map[string]*dynamodb.AttributeValue + ) + group, groupCtx := errgroup.WithContext(ctx) + for segment := 0; segment < segments; segment++ { + scanInput := &dynamodb.ScanInput{ + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + FilterExpression: expr.Filter(), + ProjectionExpression: expr.Projection(), + TableName: aws.String(tableName), + } + if segments > 1 { + scanInput.Segment = aws.Int64(int64(segment)) + scanInput.TotalSegments = aws.Int64(int64(segments)) + } + group.Go(func() error { + for { + results, scanErr := repo.dynamoDBClient.ScanWithContext(groupCtx, scanInput) + if scanErr != nil { + log.WithFields(f).WithError(scanErr).Warn("error scanning table") + return scanErr + } + mu.Lock() + items = append(items, results.Items...) + mu.Unlock() + if len(results.LastEvaluatedKey) == 0 { + return nil + } + scanInput.ExclusiveStartKey = results.LastEvaluatedKey + } + }) + } + if err := group.Wait(); err != nil { + return err + } + return dynamodbattribute.UnmarshalListOfMaps(items, out) +} diff --git a/cla-backend-go/v2/cla_search/service.go b/cla-backend-go/v2/cla_search/service.go new file mode 100644 index 000000000..3d6f7eac4 --- /dev/null +++ b/cla-backend-go/v2/cla_search/service.go @@ -0,0 +1,505 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package cla_search + +import ( + "context" + "net/url" + "sort" + "strings" + "sync" + + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/models" + "golang.org/x/sync/errgroup" +) + +const ( + sourceGitHub = "github" + sourceGitLab = "gitlab" + sourceGerrit = "gerrit" + + matchClaGroup = "claGroup" + matchProject = "project" + matchOrganization = "organization" + matchRepository = "repository" + + // DefaultLimit is the result cap applied when the caller provides no limit + DefaultLimit = 20 + + // MinSearchTermLength is the shortest search term accepted + MinSearchTermLength = 3 +) + +// match quality, lowest sorts first +const ( + rankExact = 0 + rankPrefix = 1 + rankSubstring = 2 +) + +// forgeHosts are the shared repository hosts whose hostname carries no CLA Group signal, so a +// pasted URL on one of them is matched by its path only +var forgeHosts = map[string]bool{"github.com": true, "www.github.com": true, "gitlab.com": true, "www.gitlab.com": true} + +// Service interface defines the CLA Group search service +type Service interface { + Search(ctx context.Context, searchTerm string, limit int64) (*models.ClaSearchList, error) +} + +type service struct { + repo Repository +} + +// NewService creates a new instance of the CLA Group search service +func NewService(repo Repository) Service { + return &service{repo: repo} +} + +// sources holds the reference data the four search sources are matched against +type sources struct { + claGroups []*ClaGroupRow + mappings []*ProjectMappingRow + orgs []*OrgRow +} + +// Search resolves the search term against the CLA Group names, the project/foundation names, the +// linked organization names and the repository the term resolves to. Each source is searched in its +// own goroutine and the results are merged by CLA Group - the CLA Group is the signing unit. +func (s *service) Search(ctx context.Context, searchTerm string, limit int64) (*models.ClaSearchList, error) { + rawTerm := strings.TrimSpace(searchTerm) + term := strings.ToLower(rawTerm) + if limit <= 0 { + limit = DefaultLimit + } + + src, err := s.loadSources(ctx) + if err != nil { + return nil, err + } + sfidToClaGroups := indexProjectSFIDs(src.mappings) + + m := newMatcher() + searchers, searchCtx := errgroup.WithContext(ctx) + searchers.Go(func() error { matchClaGroupNames(src.claGroups, term, m); return nil }) + searchers.Go(func() error { matchProjectNames(src.mappings, term, m); return nil }) + searchers.Go(func() error { matchOrgNames(src.orgs, term, sfidToClaGroups, m); return nil }) + searchers.Go(func() error { return s.matchRepositories(searchCtx, rawTerm, src, sfidToClaGroups, m) }) + if err := searchers.Wait(); err != nil { + return nil, err + } + + return buildList(searchTerm, limit, m.matches, src, indexOrgs(src.orgs, sfidToClaGroups)), nil +} + +// loadSources loads the reference data of every search source in parallel +func (s *service) loadSources(ctx context.Context) (*sources, error) { + var ( + src sources + orgsMu sync.Mutex + ) + loaders, loadCtx := errgroup.WithContext(ctx) + loaders.Go(func() error { + var err error + src.claGroups, err = s.repo.GetClaGroups(loadCtx) + return err + }) + loaders.Go(func() error { + var err error + src.mappings, err = s.repo.GetProjectMappings(loadCtx) + return err + }) + for _, load := range []func(context.Context) ([]*OrgRow, error){s.repo.GetGithubOrgs, s.repo.GetGitlabOrgs, s.repo.GetGerritInstances} { + loadOrgs := load + loaders.Go(func() error { + rows, err := loadOrgs(loadCtx) + if err != nil { + return err + } + orgsMu.Lock() + defer orgsMu.Unlock() + src.orgs = append(src.orgs, rows...) + return nil + }) + } + if err := loaders.Wait(); err != nil { + return nil, err + } + return &src, nil +} + +// match is the accumulated match state of a single CLA Group +type match struct { + types map[string]bool + rank int + repoName string + repoURL string +} + +// matcher merges the matches the concurrent searchers produce +type matcher struct { + mu sync.Mutex + matches map[string]*match +} + +func newMatcher() *matcher { + return &matcher{matches: map[string]*match{}} +} + +func (m *matcher) record(claGroupID, matchType string, rank int) *match { + if claGroupID == "" { + return nil + } + m.mu.Lock() + defer m.mu.Unlock() + entry, ok := m.matches[claGroupID] + if !ok { + entry = &match{types: map[string]bool{}, rank: rank} + m.matches[claGroupID] = entry + } + entry.types[matchType] = true + if rank < entry.rank { + entry.rank = rank + } + return entry +} + +func (m *matcher) recordRepository(claGroupID, name, repoURL string) { + entry := m.record(claGroupID, matchRepository, rankExact) + if entry == nil { + return + } + m.mu.Lock() + defer m.mu.Unlock() + if entry.repoName == "" { + entry.repoName, entry.repoURL = name, repoURL + } +} + +func matchClaGroupNames(claGroups []*ClaGroupRow, term string, m *matcher) { + for _, claGroup := range claGroups { + if rank := rankOf(claGroup.Name, term); rank >= 0 { + m.record(claGroup.ClaGroupID, matchClaGroup, rank) + } + } +} + +func matchProjectNames(mappings []*ProjectMappingRow, term string, m *matcher) { + for _, mapping := range mappings { + if rank := bestRank(term, mapping.ProjectName, mapping.FoundationName); rank >= 0 { + m.record(mapping.ClaGroupID, matchProject, rank) + } + } +} + +func matchOrgNames(orgs []*OrgRow, term string, sfidToClaGroups map[string][]string, m *matcher) { + host := hostOf(term) + for _, org := range orgs { + rank := orgRank(org, term, host) + if rank < 0 { + continue + } + for _, claGroupID := range org.claGroupIDs(sfidToClaGroups) { + m.record(claGroupID, matchOrganization, rank) + } + } +} + +// orgRank matches the term against the organization name, its URL, and - for a self-hosted +// instance such as a Gerrit server - the hostname of a pasted URL under it +func orgRank(org *OrgRow, term, host string) int { + if rank := rankOf(org.Name, term); rank >= 0 { + return rank + } + if org.URL == "" { + return -1 + } + if strings.Contains(urlSignal(org.URL), term) { + return rankSubstring + } + if host != "" && !forgeHosts[host] && hostOf(org.URL) == host { + return rankSubstring + } + return -1 +} + +// matchRepositories resolves a pasted repository URL or "owner/repo" path to the CLA Group owning +// that repository, and matches its owner segment against the linked organizations - which is the +// only signal available for an auto-enabled organization, whose repositories carry no records +func (s *service) matchRepositories(ctx context.Context, rawTerm string, src *sources, sfidToClaGroups map[string][]string, m *matcher) error { + path := repositoryPath(rawTerm) + if path == "" { + return nil + } + owner := path[:strings.Index(path, "/")] + + ownerOrgs := orgsNamed(src.orgs, owner) + for _, org := range ownerOrgs { + for _, claGroupID := range org.claGroupIDs(sfidToClaGroups) { + m.record(claGroupID, matchOrganization, rankExact) + } + } + + repos, err := s.repo.GetRepositoriesByName(ctx, nameVariants(path)) + if err != nil { + return err + } + // the repository-name-index GSI is keyed on the case-preserved name, so a lower-cased paste of a + // mixed-case repository misses it - the owner's repositories are then listed through the + // organization GSI and compared case-insensitively + if len(repos) == 0 && len(ownerOrgs) > 0 { + repos, err = s.repo.GetRepositoriesByOrganization(ctx, organizationNames(ownerOrgs)) + if err != nil { + return err + } + } + + lowerPath := strings.ToLower(path) + for _, repo := range repos { + if strings.ToLower(repo.Name) == lowerPath { + m.recordRepository(repo.ClaGroupID, repo.Name, repo.URL) + } + } + return nil +} + +// orgsNamed returns the organizations whose name is the given name, compared case-insensitively +func orgsNamed(orgs []*OrgRow, name string) []*OrgRow { + var matched []*OrgRow + for _, org := range orgs { + if org.Name != "" && strings.EqualFold(org.Name, name) { + matched = append(matched, org) + } + } + return matched +} + +func organizationNames(orgs []*OrgRow) []string { + seen := map[string]bool{} + names := make([]string, 0, len(orgs)) + for _, org := range orgs { + if !seen[org.Name] { + seen[org.Name] = true + names = append(names, org.Name) + } + } + return names +} + +// claGroupIDs returns the CLA Groups the organization is linked to - Gerrit instances reference the +// CLA Group directly, GitHub organizations and GitLab groups reference it by project SFID +func (o *OrgRow) claGroupIDs(sfidToClaGroups map[string][]string) []string { + if o.ClaGroupID != "" { + return []string{o.ClaGroupID} + } + return sfidToClaGroups[o.ProjectSFID] +} + +func indexProjectSFIDs(mappings []*ProjectMappingRow) map[string][]string { + index := map[string][]string{} + for _, mapping := range mappings { + if mapping.ProjectSFID != "" && mapping.ClaGroupID != "" { + index[mapping.ProjectSFID] = append(index[mapping.ProjectSFID], mapping.ClaGroupID) + } + } + return index +} + +func indexOrgs(orgs []*OrgRow, sfidToClaGroups map[string][]string) map[string][]models.ClaSearchOrg { + index := map[string][]models.ClaSearchOrg{} + for _, org := range orgs { + for _, claGroupID := range org.claGroupIDs(sfidToClaGroups) { + index[claGroupID] = append(index[claGroupID], models.ClaSearchOrg{Name: org.Name, Source: org.Source, URL: orgURL(org)}) + } + } + return index +} + +// orgURL is the organization URL, derived for a GitHub organization - the github-orgs records carry none +func orgURL(org *OrgRow) string { + if org.URL == "" && org.Source == sourceGitHub && org.Name != "" { + return "https://github.com/" + org.Name + } + return org.URL +} + +func buildList(searchTerm string, limit int64, matches map[string]*match, src *sources, orgsByClaGroup map[string][]models.ClaSearchOrg) *models.ClaSearchList { + claGroupByID := map[string]*ClaGroupRow{} + for _, claGroup := range src.claGroups { + claGroupByID[claGroup.ClaGroupID] = claGroup + } + mappingsByClaGroup := map[string][]*ProjectMappingRow{} + for _, mapping := range src.mappings { + mappingsByClaGroup[mapping.ClaGroupID] = append(mappingsByClaGroup[mapping.ClaGroupID], mapping) + } + + results := make([]models.ClaSearchResult, 0, len(matches)) + for claGroupID, m := range matches { + results = append(results, buildResult(claGroupID, m, claGroupByID[claGroupID], mappingsByClaGroup[claGroupID], orgsByClaGroup[claGroupID])) + } + sort.Slice(results, func(i, j int) bool { + if ri, rj := matches[results[i].ClaGroupID].rank, matches[results[j].ClaGroupID].rank; ri != rj { + return ri < rj + } + if a, b := displayName(results[i]), displayName(results[j]); a != b { + return a < b + } + return results[i].ClaGroupID < results[j].ClaGroupID + }) + + truncated := int64(len(results)) > limit + if truncated { + results = results[:limit] + } + + return &models.ClaSearchList{ + SearchTerm: searchTerm, + ResultCount: int64(len(results)), + Truncated: truncated, + Results: results, + } +} + +func buildResult(claGroupID string, m *match, claGroup *ClaGroupRow, mappings []*ProjectMappingRow, orgs []models.ClaSearchOrg) models.ClaSearchResult { + result := models.ClaSearchResult{ + ClaGroupID: claGroupID, + MatchTypes: sortedKeys(m.types), + MatchedRepositoryName: m.repoName, + MatchedRepositoryURL: m.repoURL, + Organizations: sortOrgs(orgs), + } + if claGroup != nil { + result.ClaGroupName = claGroup.Name + result.ProjectExternalID = claGroup.ExternalID + result.IclaEnabled = claGroup.IclaEnabled + result.CclaEnabled = claGroup.CclaEnabled + } + + // A foundation-level CLA Group is marked by a mapping whose ProjectSFID equals its + // FoundationSFID (the projects_cla_groups convention) and resolves to its foundation; a single + // project-level mapping resolves to that project. Several project-level mappings with no + // foundation marker are left unresolved rather than picking an arbitrary one. + for _, mapping := range mappings { + if result.ClaGroupName == "" { + result.ClaGroupName = mapping.ClaGroupName + } + if result.FoundationSFID == "" { + result.FoundationSFID = mapping.FoundationSFID + } + if mapping.FoundationSFID != "" && mapping.FoundationSFID == mapping.ProjectSFID { + result.ProjectSFID, result.ProjectName = mapping.FoundationSFID, mapping.FoundationName + return result + } + } + if len(mappings) == 1 { + result.ProjectSFID, result.ProjectName = mappings[0].ProjectSFID, mappings[0].ProjectName + } + return result +} + +func displayName(result models.ClaSearchResult) string { + if result.ProjectName != "" { + return strings.ToLower(result.ProjectName) + } + return strings.ToLower(result.ClaGroupName) +} + +func sortOrgs(orgs []models.ClaSearchOrg) []models.ClaSearchOrg { + if orgs == nil { + return []models.ClaSearchOrg{} + } + sort.Slice(orgs, func(i, j int) bool { + if orgs[i].Source != orgs[j].Source { + return orgs[i].Source < orgs[j].Source + } + return orgs[i].Name < orgs[j].Name + }) + return orgs +} + +func sortedKeys(set map[string]bool) []string { + keys := make([]string, 0, len(set)) + for key := range set { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +// rankOf scores how well the already lower-cased term matches value - lower is better, -1 is no match +func rankOf(value, term string) int { + value = strings.ToLower(value) + switch { + case value == "" || term == "": + return -1 + case value == term: + return rankExact + case strings.HasPrefix(value, term): + return rankPrefix + case strings.Contains(value, term): + return rankSubstring + default: + return -1 + } +} + +func bestRank(term string, values ...string) int { + best := -1 + for _, value := range values { + if rank := rankOf(value, term); rank >= 0 && (best < 0 || rank < best) { + best = rank + } + } + return best +} + +func hostOf(rawURL string) string { + parsed, err := url.Parse(strings.ToLower(rawURL)) + if err != nil { + return "" + } + return parsed.Hostname() +} + +// repositoryPath derives the full repository name the term addresses - the path of a pasted +// repository URL, or the term itself when it looks like an "owner/repo" path - and is empty when +// the term addresses no repository +func repositoryPath(term string) string { + path := term + if strings.Contains(term, "://") { + parsed, err := url.Parse(term) + if err != nil || parsed.Hostname() == "" { + return "" + } + path = parsed.Path + } + path = strings.Trim(path, "/") + path = strings.TrimSuffix(path, ".git") + path = strings.TrimPrefix(path, "groups/") + if !strings.Contains(path, "/") || strings.ContainsAny(path, " \t") { + return "" + } + return path +} + +// nameVariants are the repository names looked up on the case-preserved repository-name-index GSI +func nameVariants(path string) []string { + if lower := strings.ToLower(path); lower != path { + return []string{path, lower} + } + return []string{path} +} + +// urlSignal is the part of an organization URL that identifies the organization - the host of a +// shared forge is the same for every organization hosted on it and carries no signal, while the +// host of a self-hosted instance such as a Gerrit server is the only thing that does +func urlSignal(rawURL string) string { + parsed, err := url.Parse(strings.ToLower(rawURL)) + if err != nil { + return strings.ToLower(rawURL) + } + if forgeHosts[parsed.Hostname()] { + return strings.Trim(parsed.Path, "/") + } + return parsed.Hostname() + parsed.Path +} diff --git a/cla-backend-go/v2/cla_search/service_test.go b/cla-backend-go/v2/cla_search/service_test.go new file mode 100644 index 000000000..ef152d1d3 --- /dev/null +++ b/cla-backend-go/v2/cla_search/service_test.go @@ -0,0 +1,451 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package cla_search + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeRepo struct { + claGroups []*ClaGroupRow + mappings []*ProjectMappingRow + github []*OrgRow + gitlab []*OrgRow + gerrit []*OrgRow + repos map[string][]*RepositoryRow + orgRepos map[string][]*RepositoryRow + + failOn string + + mu sync.Mutex + repoQueries [][]string + orgQueries [][]string + loaderCalls int + repoCalls int + callSequence []string +} + +func (f *fakeRepo) note(name string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.callSequence = append(f.callSequence, name) + f.loaderCalls++ + if f.failOn == name { + return errors.New("boom: " + name) + } + return nil +} + +func (f *fakeRepo) GetClaGroups(_ context.Context) ([]*ClaGroupRow, error) { + if err := f.note("claGroups"); err != nil { + return nil, err + } + return f.claGroups, nil +} + +func (f *fakeRepo) GetProjectMappings(_ context.Context) ([]*ProjectMappingRow, error) { + if err := f.note("mappings"); err != nil { + return nil, err + } + return f.mappings, nil +} + +func (f *fakeRepo) GetGithubOrgs(_ context.Context) ([]*OrgRow, error) { + if err := f.note("github"); err != nil { + return nil, err + } + return f.github, nil +} + +func (f *fakeRepo) GetGitlabOrgs(_ context.Context) ([]*OrgRow, error) { + if err := f.note("gitlab"); err != nil { + return nil, err + } + return f.gitlab, nil +} + +func (f *fakeRepo) GetGerritInstances(_ context.Context) ([]*OrgRow, error) { + if err := f.note("gerrit"); err != nil { + return nil, err + } + return f.gerrit, nil +} + +func (f *fakeRepo) GetRepositoriesByName(_ context.Context, names []string) ([]*RepositoryRow, error) { + f.mu.Lock() + f.repoQueries = append(f.repoQueries, names) + f.repoCalls++ + f.mu.Unlock() + if f.failOn == "repositories" { + return nil, errors.New("boom: repositories") + } + var rows []*RepositoryRow + for _, name := range names { + rows = append(rows, f.repos[name]...) + } + return rows, nil +} + +func (f *fakeRepo) GetRepositoriesByOrganization(_ context.Context, organizationNames []string) ([]*RepositoryRow, error) { + f.mu.Lock() + f.orgQueries = append(f.orgQueries, organizationNames) + f.mu.Unlock() + if f.failOn == "organizationRepositories" { + return nil, errors.New("boom: organizationRepositories") + } + var rows []*RepositoryRow + for _, name := range organizationNames { + rows = append(rows, f.orgRepos[name]...) + } + return rows, nil +} + +// sampleRepo mirrors the production shape: two CLA groups behind GitHub orgs, one foundation-level +// CLA group behind a Gerrit instance, and one behind a GitLab group +func sampleRepo() *fakeRepo { + return &fakeRepo{ + claGroups: []*ClaGroupRow{ + {ClaGroupID: "cg-kube", Name: "Kubernetes CLA", ExternalID: "a09-kube", IclaEnabled: true, CclaEnabled: true}, + {ClaGroupID: "cg-otio", Name: "OpenTimelineIO CLA", ExternalID: "a09-otio", CclaEnabled: true}, + {ClaGroupID: "cg-onap", Name: "ONAP CLA", ExternalID: "a09-onap-f", IclaEnabled: true}, + {ClaGroupID: "cg-orphan", Name: "Kubernetes Edge CLA"}, + }, + mappings: []*ProjectMappingRow{ + {ClaGroupID: "cg-kube", ClaGroupName: "Kubernetes CLA", ProjectSFID: "sfid-kube", ProjectName: "Kubernetes", FoundationSFID: "sfid-cncf", FoundationName: "CNCF"}, + {ClaGroupID: "cg-otio", ClaGroupName: "OpenTimelineIO CLA", ProjectSFID: "sfid-otio", ProjectName: "OpenTimelineIO", FoundationSFID: "sfid-aswf", FoundationName: "Academy Software Foundation"}, + {ClaGroupID: "cg-onap", ClaGroupName: "ONAP CLA", ProjectSFID: "sfid-onap-f", ProjectName: "ONAP Foundation Level", FoundationSFID: "sfid-onap-f", FoundationName: "ONAP"}, + {ClaGroupID: "cg-multi", ClaGroupName: "Shared CLA", ProjectSFID: "sfid-a", ProjectName: "Shared Project A", FoundationSFID: "sfid-root"}, + {ClaGroupID: "cg-multi", ClaGroupName: "Shared CLA", ProjectSFID: "sfid-b", ProjectName: "Shared Project B", FoundationSFID: "sfid-root"}, + }, + github: []*OrgRow{ + {Name: "kubernetes", Source: sourceGitHub, ProjectSFID: "sfid-kube"}, + {Name: "kubernetes-sigs", Source: sourceGitHub, ProjectSFID: "sfid-kube"}, + {Name: "OpenTimelineIO", Source: sourceGitHub, ProjectSFID: "sfid-otio"}, + }, + gitlab: []*OrgRow{ + {Name: "onap", URL: "https://gitlab.com/groups/onap", Source: sourceGitLab, ProjectSFID: "sfid-onap-f"}, + }, + gerrit: []*OrgRow{ + {Name: "ONAP", URL: "https://gerrit.onap.org", Source: sourceGerrit, ClaGroupID: "cg-onap"}, + }, + repos: map[string][]*RepositoryRow{ + "OpenTimelineIO/OpenTimelineIO-Java-Bindings": {{Name: "OpenTimelineIO/OpenTimelineIO-Java-Bindings", URL: "https://github.com/OpenTimelineIO/OpenTimelineIO-Java-Bindings", Type: sourceGitHub, ClaGroupID: "cg-otio"}}, + "onap/oom/oom": {{Name: "onap/oom/oom", URL: "https://gitlab.com/onap/oom/oom", Type: sourceGitLab, ClaGroupID: "cg-onap"}}, + }, + orgRepos: map[string][]*RepositoryRow{ + "OpenTimelineIO": { + {Name: "OpenTimelineIO/OpenTimelineIO-Java-Bindings", URL: "https://github.com/OpenTimelineIO/OpenTimelineIO-Java-Bindings", Type: sourceGitHub, ClaGroupID: "cg-otio"}, + {Name: "OpenTimelineIO/otio-plugin-template", URL: "https://github.com/OpenTimelineIO/otio-plugin-template", Type: sourceGitHub, ClaGroupID: "cg-otio"}, + }, + }, + } +} + +func resultByID(list *models.ClaSearchList, claGroupID string) *models.ClaSearchResult { + for i := range list.Results { + if list.Results[i].ClaGroupID == claGroupID { + return &list.Results[i] + } + } + return nil +} + +func ids(list *models.ClaSearchList) []string { + out := make([]string, 0, len(list.Results)) + for i := range list.Results { + out = append(out, list.Results[i].ClaGroupID) + } + return out +} + +func TestSearchByClaGroupName(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), "kubernetes cla", 0) + require.NoError(t, err) + assert.Equal(t, []string{"cg-kube"}, ids(list)) + assert.Equal(t, int64(1), list.ResultCount) + assert.False(t, list.Truncated) + + kube := resultByID(list, "cg-kube") + require.NotNil(t, kube) + assert.Equal(t, []string{matchClaGroup}, kube.MatchTypes) + assert.Equal(t, "Kubernetes CLA", kube.ClaGroupName) + assert.Equal(t, "Kubernetes", kube.ProjectName) + assert.Equal(t, "sfid-kube", kube.ProjectSFID) + assert.Equal(t, "sfid-cncf", kube.FoundationSFID) + assert.Equal(t, "a09-kube", kube.ProjectExternalID) + assert.True(t, kube.IclaEnabled) + assert.True(t, kube.CclaEnabled) +} + +func TestSearchByProjectName(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), "opentimelineio", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-otio"}, ids(list)) + assert.Equal(t, []string{matchClaGroup, matchOrganization, matchProject}, list.Results[0].MatchTypes) +} + +func TestSearchByFoundationName(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), "academy software", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-otio"}, ids(list)) + assert.Equal(t, []string{matchProject}, list.Results[0].MatchTypes) +} + +func TestSearchByOrgNameCarriesProvenance(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), "kubernetes-sigs", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-kube"}, ids(list)) + assert.Equal(t, []string{matchOrganization}, list.Results[0].MatchTypes) + // every organization linked to the CLA Group is returned, and the GitHub URL is derived + assert.Equal(t, []models.ClaSearchOrg{ + {Name: "kubernetes", Source: sourceGitHub, URL: "https://github.com/kubernetes"}, + {Name: "kubernetes-sigs", Source: sourceGitHub, URL: "https://github.com/kubernetes-sigs"}, + }, list.Results[0].Organizations) +} + +func TestSearchGitlabAndGerritProvenanceAreReturned(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), "onap", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-onap"}, ids(list)) + onap := list.Results[0] + assert.Equal(t, []string{matchClaGroup, matchOrganization, matchProject}, onap.MatchTypes) + assert.Equal(t, []models.ClaSearchOrg{ + {Name: "ONAP", Source: sourceGerrit, URL: "https://gerrit.onap.org"}, + {Name: "onap", Source: sourceGitLab, URL: "https://gitlab.com/groups/onap"}, + }, onap.Organizations) + // foundation-level CLA group resolves to its foundation + assert.Equal(t, "ONAP", onap.ProjectName) + assert.Equal(t, "sfid-onap-f", onap.ProjectSFID) +} + +func TestSearchResolvesPastedRepoURL(t *testing.T) { + for _, term := range []string{ + "https://github.com/OpenTimelineIO/OpenTimelineIO-Java-Bindings", + "https://github.com/OpenTimelineIO/OpenTimelineIO-Java-Bindings.git", + "https://github.com/OpenTimelineIO/OpenTimelineIO-Java-Bindings/", + "OpenTimelineIO/OpenTimelineIO-Java-Bindings", + } { + t.Run(term, func(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), term, 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-otio"}, ids(list)) + assert.Contains(t, list.Results[0].MatchTypes, matchRepository) + assert.Equal(t, "OpenTimelineIO/OpenTimelineIO-Java-Bindings", list.Results[0].MatchedRepositoryName) + assert.Equal(t, "https://github.com/OpenTimelineIO/OpenTimelineIO-Java-Bindings", list.Results[0].MatchedRepositoryURL) + }) + } +} + +func TestSearchResolvesNestedGitlabRepoURL(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), "https://gitlab.com/onap/oom/oom", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-onap"}, ids(list)) + assert.Contains(t, list.Results[0].MatchTypes, matchRepository) +} + +func TestSearchResolvesGerritHostURL(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), "https://gerrit.onap.org/r/aai/aai-common", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-onap"}, ids(list)) + assert.Equal(t, []string{matchOrganization}, list.Results[0].MatchTypes) +} + +func TestSearchDoesNotMatchEveryGroupOnForgeHost(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), "https://github.com/unknown-org/unknown-repo", 0) + require.NoError(t, err) + assert.Empty(t, list.Results) + assert.Equal(t, int64(0), list.ResultCount) + assert.NotNil(t, list.Results) +} + +func TestSearchResolvesLowerCasedPasteOfMixedCaseRepoURL(t *testing.T) { + repo := sampleRepo() + svc := NewService(repo) + list, err := svc.Search(context.Background(), "https://github.com/opentimelineio/opentimelineio-java-bindings", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-otio"}, ids(list)) + assert.Contains(t, list.Results[0].MatchTypes, matchRepository) + assert.Equal(t, "OpenTimelineIO/OpenTimelineIO-Java-Bindings", list.Results[0].MatchedRepositoryName) + assert.Equal(t, [][]string{{"OpenTimelineIO"}}, repo.orgQueries) +} + +func TestSearchOwnerOfOrganizationMatchesWhenRepositoryHasNoRecord(t *testing.T) { + repo := sampleRepo() + svc := NewService(repo) + list, err := svc.Search(context.Background(), "https://github.com/kubernetes/a-brand-new-repo", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-kube"}, ids(list)) + assert.Equal(t, []string{matchOrganization}, list.Results[0].MatchTypes) + assert.Empty(t, list.Results[0].MatchedRepositoryName) +} + +func TestSearchForgeNameDoesNotMatchEveryOrganizationOnIt(t *testing.T) { + for _, term := range []string{"gitlab", "github", "gitlab.com"} { + t.Run(term, func(t *testing.T) { + list, err := NewService(sampleRepo()).Search(context.Background(), term, 0) + require.NoError(t, err) + assert.Empty(t, list.Results) + }) + } +} + +func TestSearchMatchesSelfHostedInstanceHost(t *testing.T) { + list, err := NewService(sampleRepo()).Search(context.Background(), "gerrit.onap.org", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-onap"}, ids(list)) +} + +func TestSearchQueriesRepositoryNameCasePreservedAndLowered(t *testing.T) { + repo := sampleRepo() + svc := NewService(repo) + _, err := svc.Search(context.Background(), "https://github.com/OpenTimelineIO/OpenTimelineIO-Java-Bindings", 0) + require.NoError(t, err) + require.Equal(t, [][]string{{ + "OpenTimelineIO/OpenTimelineIO-Java-Bindings", + "opentimelineio/opentimelineio-java-bindings", + }}, repo.repoQueries) +} + +func TestSearchSkipsRepositoryLookupWithoutAPath(t *testing.T) { + repo := sampleRepo() + svc := NewService(repo) + _, err := svc.Search(context.Background(), "kubernetes", 0) + require.NoError(t, err) + assert.Zero(t, repo.repoCalls) +} + +func TestSearchAmbiguousMultiProjectClaGroupOmitsProjectFields(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), "shared", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-multi"}, ids(list)) + assert.Empty(t, list.Results[0].ProjectName) + assert.Empty(t, list.Results[0].ProjectSFID) + assert.Equal(t, "sfid-root", list.Results[0].FoundationSFID) + // the CLA group record is absent from the projects table, the mapping supplies the name + assert.Equal(t, "Shared CLA", list.Results[0].ClaGroupName) + assert.Equal(t, []models.ClaSearchOrg{}, list.Results[0].Organizations) +} + +func TestSearchIsCaseInsensitiveAndTrimmed(t *testing.T) { + svc := NewService(sampleRepo()) + list, err := svc.Search(context.Background(), " KUBERNETES-SIGS ", 0) + require.NoError(t, err) + assert.Equal(t, []string{"cg-kube"}, ids(list)) + assert.Equal(t, " KUBERNETES-SIGS ", list.SearchTerm) +} + +func TestSearchRanksExactBeforePrefixBeforeSubstring(t *testing.T) { + repo := &fakeRepo{claGroups: []*ClaGroupRow{ + {ClaGroupID: "cg-sub", Name: "The Zeta Project"}, + {ClaGroupID: "cg-exact", Name: "zeta"}, + {ClaGroupID: "cg-prefix", Name: "Zeta Networking"}, + }} + list, err := NewService(repo).Search(context.Background(), "zeta", 0) + require.NoError(t, err) + assert.Equal(t, []string{"cg-exact", "cg-prefix", "cg-sub"}, ids(list)) +} + +func TestSearchTruncatesAtLimit(t *testing.T) { + repo := &fakeRepo{} + for i := 0; i < 5; i++ { + repo.claGroups = append(repo.claGroups, &ClaGroupRow{ClaGroupID: fmt.Sprintf("cg-%d", i), Name: fmt.Sprintf("Zeta %d CLA", i)}) + } + list, err := NewService(repo).Search(context.Background(), "zeta", 3) + require.NoError(t, err) + assert.True(t, list.Truncated) + assert.Equal(t, int64(3), list.ResultCount) + assert.Equal(t, []string{"cg-0", "cg-1", "cg-2"}, ids(list)) +} + +func TestSearchNotTruncatedAtExactlyLimit(t *testing.T) { + repo := &fakeRepo{claGroups: []*ClaGroupRow{ + {ClaGroupID: "cg-0", Name: "Zeta A CLA"}, + {ClaGroupID: "cg-1", Name: "Zeta B CLA"}, + }} + list, err := NewService(repo).Search(context.Background(), "zeta", 2) + require.NoError(t, err) + assert.False(t, list.Truncated) + assert.Equal(t, int64(2), list.ResultCount) +} + +func TestSearchDefaultsLimit(t *testing.T) { + repo := &fakeRepo{} + for i := 0; i < DefaultLimit+1; i++ { + repo.claGroups = append(repo.claGroups, &ClaGroupRow{ClaGroupID: fmt.Sprintf("cg-%02d", i), Name: fmt.Sprintf("Zeta %02d CLA", i)}) + } + list, err := NewService(repo).Search(context.Background(), "zeta", 0) + require.NoError(t, err) + assert.True(t, list.Truncated) + assert.Equal(t, int64(DefaultLimit), list.ResultCount) +} + +func TestSearchNoMatchReturnsEmptyList(t *testing.T) { + list, err := NewService(sampleRepo()).Search(context.Background(), "nothing-matches-this", 0) + require.NoError(t, err) + assert.Equal(t, int64(0), list.ResultCount) + assert.False(t, list.Truncated) + assert.NotNil(t, list.Results) +} + +func TestSearchPropagatesSourceErrors(t *testing.T) { + for _, source := range []string{"claGroups", "mappings", "github", "gitlab", "gerrit", "repositories"} { + t.Run(source, func(t *testing.T) { + repo := sampleRepo() + repo.failOn = source + list, err := NewService(repo).Search(context.Background(), "OpenTimelineIO/OpenTimelineIO-Java-Bindings", 0) + require.Error(t, err) + assert.Contains(t, err.Error(), "boom: "+source) + assert.Nil(t, list) + }) + } +} + +func TestSearchRunsSourcesConcurrently(t *testing.T) { + repo := sampleRepo() + _, err := NewService(repo).Search(context.Background(), "onap/oom/oom", 0) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"claGroups", "mappings", "github", "gitlab", "gerrit"}, repo.callSequence) + assert.Equal(t, 1, repo.repoCalls) +} + +func TestRepositoryPath(t *testing.T) { + for _, tc := range []struct { + term string + expected string + variants []string + }{ + {"kubernetes", "", nil}, + {"has space/repo", "", nil}, + {"not-a-url://", "", nil}, + {"Owner/Repo", "Owner/Repo", []string{"Owner/Repo", "owner/repo"}}, + {"owner/repo", "owner/repo", []string{"owner/repo"}}, + {"https://gitlab.com/groups/onap", "", nil}, + {"https://gitlab.com/onap/oom/oom", "onap/oom/oom", []string{"onap/oom/oom"}}, + {"https://github.com/Owner/Repo.git", "Owner/Repo", []string{"Owner/Repo", "owner/repo"}}, + } { + path := repositoryPath(tc.term) + assert.Equal(t, tc.expected, path, tc.term) + if path != "" { + assert.Equal(t, tc.variants, nameVariants(path), tc.term) + } + } +} diff --git a/utils/cla_search.sh b/utils/cla_search.sh new file mode 100755 index 000000000..bc78e832f --- /dev/null +++ b/utils/cla_search.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Copyright The Linux Foundation and each contributor to CommunityBridge. +# SPDX-License-Identifier: MIT + +# Calls the CLA Group search API (GET /v4/cla-group/search) through lfx-gateway and reports the HTTP status and total time. +# SEARCH_TERM (or 1st arg): the term to search for - a CLA Group name, project/foundation name, organization name, or a pasted repository URL or "owner/repo" path (min 3 characters). +# LIMIT: the result cap (1-100, default 20 server-side). +# TOKEN: bearer access token (env, or ./cla_search.token.secret / ./my_clas.token.secret / ./auth0.token.secret). Get one with ~/get_oauth_token.sh (dev) or ~/get_oauth_token_prod.sh (prod). +# STAGE: dev (default) | test | staging | prod - selects the api-gw host. +# Local mode (against a standalone backend, bypassing the gateway): set PRINCIPAL to the token username (or pass a raw base64 X_ACL). Defaults API_URL to http://localhost:8080. +# RUNS: when >1, repeats the call that many times and reports the min/p50/p95/max server time (FR-001a's < 300 ms p95 budget); the body is printed only for the first run. +# Examples: +# ./utils/cla_search.sh kubernetes +# SEARCH_TERM="https://github.com/OpenTimelineIO/OpenTimelineIO-Java-Bindings" ./utils/cla_search.sh +# STAGE=prod TOKEN="$(~/get_oauth_token_prod.sh)" RUNS=30 ./utils/cla_search.sh onap +# PRINCIPAL=lgryglicki ./utils/cla_search.sh kube # local standalone backend + +if [ -n "$PRINCIPAL" ] && [ -z "$X_ACL" ] +then + X_ACL="$(printf '{"user_name":"%s","email":"%s","isAdmin":false,"allowed":true}' "$PRINCIPAL" "${PRINCIPAL_EMAIL:-$PRINCIPAL}" | base64 | tr -d '\n')" +fi + +if [ -n "$X_ACL" ] +then + auth=(-H "X-ACL: ${X_ACL}") + [ -z "$API_URL" ] && API_URL="http://localhost:${PORT:-8080}" +else + for f in ./cla_search.token.secret ./my_clas.token.secret ./auth0.token.secret + do + [ -n "$TOKEN" ] && break + [ -f "$f" ] && TOKEN="$(cat "$f")" + done + if [ -z "$TOKEN" ] + then + echo "$0: TOKEN not set - run ~/get_oauth_token.sh (dev) or ~/get_oauth_token_prod.sh (prod) and export TOKEN (or use PRINCIPAL=... for local mode)" + exit 1 + fi + auth=(-H "Authorization: Bearer ${TOKEN}") + [ -z "$STAGE" ] && STAGE=dev + case "$STAGE" in + prod) GW="https://api-gw.platform.linuxfoundation.org" ;; + staging) GW="https://api-gw.staging.platform.linuxfoundation.org" ;; + test) GW="https://api-gw.test.platform.linuxfoundation.org" ;; + dev) GW="https://api-gw.dev.platform.linuxfoundation.org" ;; + *) echo "$0: unknown STAGE '$STAGE'"; exit 2 ;; + esac + [ -z "$API_URL" ] && API_URL="${GW}/cla-service" +fi + +[ -z "$SEARCH_TERM" ] && SEARCH_TERM="$1" +if [ -z "$SEARCH_TERM" ] +then + echo "$0: SEARCH_TERM not set - pass it as the first argument or in the environment" + exit 3 +fi + +URL="${API_URL}/v4/cla-group/search" +args=(--data-urlencode "searchTerm=${SEARCH_TERM}") +[ -n "$LIMIT" ] && args+=(--data-urlencode "limit=${LIMIT}") + +if [ -n "$DEBUG" ] +then + echo "curl -sS -G -XGET ${auth[0]} '' -H 'Content-Type: application/json' ${args[*]} '${URL}'" +fi + +[ -z "$RUNS" ] && RUNS=1 +body="$(mktemp)" +times="$(mktemp)" +for i in $(seq 1 "$RUNS") +do + timing="$(curl -sS -G -XGET "${auth[@]}" -H "Content-Type: application/json" "${args[@]}" -w '%{http_code} %{time_total}' -o "$body" "$URL")" + code="${timing% *}" + secs="${timing#* }" + echo "$secs" >> "$times" + if [ "$i" = "1" ] + then + if command -v jq >/dev/null 2>&1 + then + jq -r '.' < "$body" 2>/dev/null || cat "$body" + else + cat "$body" + fi + echo + fi + [ "$RUNS" = "1" ] && echo "HTTP ${code} in ${secs}s" +done + +if [ "$RUNS" != "1" ] +then + sort -n "$times" | awk -v runs="$RUNS" '{t[NR]=$1} END { + p50=t[int((NR+1)*0.50+0.5)]; p95=t[int((NR+1)*0.95+0.5)] + printf "runs=%d min=%.3fs p50=%.3fs p95=%.3fs max=%.3fs\n", runs, t[1], p50, p95, t[NR] + }' +fi +rm -f "$body" "$times" From 081498d9767c2ddc68499d64aa407687ac70fadb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Gryglicki?= Date: Wed, 19 Aug 2026 05:35:53 +0000 Subject: [PATCH 2/7] Search CLA group API - 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) Assisted by [Claude](https://claude.ai) --- cla-backend-go/swagger/cla.v2.yaml | 4 +- cla-backend-go/v2/cla_search/handlers_test.go | 6 +++ cla-backend-go/v2/cla_search/service.go | 39 +++++++++++++------ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/cla-backend-go/swagger/cla.v2.yaml b/cla-backend-go/swagger/cla.v2.yaml index eec9d0ab7..87c49bd09 100644 --- a/cla-backend-go/swagger/cla.v2.yaml +++ b/cla-backend-go/swagger/cla.v2.yaml @@ -2865,7 +2865,7 @@ paths: /cla-group/search: get: summary: Search CLA Groups - description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit + description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422 operationId: searchClaGroups parameters: - $ref: "#/parameters/x-request-id" @@ -2889,6 +2889,8 @@ paths: $ref: '#/responses/unauthorized' '403': $ref: '#/responses/forbidden' + '422': + $ref: '#/responses/unprocessable-entity' '500': $ref: '#/responses/internal-server-error' tags: diff --git a/cla-backend-go/v2/cla_search/handlers_test.go b/cla-backend-go/v2/cla_search/handlers_test.go index 8ccce798e..9912cd2d8 100644 --- a/cla-backend-go/v2/cla_search/handlers_test.go +++ b/cla-backend-go/v2/cla_search/handlers_test.go @@ -55,6 +55,12 @@ func TestHandlerUnauthorizedWithoutPrincipal(t *testing.T) { } } +func TestHandlerAcceptsAdminPrincipalWithoutUsername(t *testing.T) { + svc := &stubService{list: &models.ClaSearchList{SearchTerm: "kubernetes"}} + assert.Equal(t, http.StatusOK, invoke(t, svc, &auth.User{ACL: auth.ACL{Admin: true}}, "kubernetes").Code) + assert.Equal(t, "kubernetes", svc.term) +} + func TestHandlerBadRequestOnWhitespaceOnlyTerm(t *testing.T) { svc := &stubService{} assert.Equal(t, http.StatusBadRequest, invoke(t, svc, &auth.User{UserName: "jdoe"}, " ").Code) diff --git a/cla-backend-go/v2/cla_search/service.go b/cla-backend-go/v2/cla_search/service.go index 3d6f7eac4..31283b05f 100644 --- a/cla-backend-go/v2/cla_search/service.go +++ b/cla-backend-go/v2/cla_search/service.go @@ -73,8 +73,26 @@ func (s *service) Search(ctx context.Context, searchTerm string, limit int64) (* limit = DefaultLimit } - src, err := s.loadSources(ctx) - if err != nil { + // the reference data of every source and the repository the term addresses are fetched together + var ( + src *sources + repos []*RepositoryRow + ) + path := repositoryPath(rawTerm) + fetch, fetchCtx := errgroup.WithContext(ctx) + fetch.Go(func() error { + var err error + src, err = s.loadSources(fetchCtx) + return err + }) + if path != "" { + fetch.Go(func() error { + var err error + repos, err = s.repo.GetRepositoriesByName(fetchCtx, nameVariants(path)) + return err + }) + } + if err := fetch.Wait(); err != nil { return nil, err } sfidToClaGroups := indexProjectSFIDs(src.mappings) @@ -84,7 +102,7 @@ func (s *service) Search(ctx context.Context, searchTerm string, limit int64) (* searchers.Go(func() error { matchClaGroupNames(src.claGroups, term, m); return nil }) searchers.Go(func() error { matchProjectNames(src.mappings, term, m); return nil }) searchers.Go(func() error { matchOrgNames(src.orgs, term, sfidToClaGroups, m); return nil }) - searchers.Go(func() error { return s.matchRepositories(searchCtx, rawTerm, src, sfidToClaGroups, m) }) + searchers.Go(func() error { return s.matchRepositories(searchCtx, path, repos, src, sfidToClaGroups, m) }) if err := searchers.Wait(); err != nil { return nil, err } @@ -223,11 +241,11 @@ func orgRank(org *OrgRow, term, host string) int { return -1 } -// matchRepositories resolves a pasted repository URL or "owner/repo" path to the CLA Group owning -// that repository, and matches its owner segment against the linked organizations - which is the -// only signal available for an auto-enabled organization, whose repositories carry no records -func (s *service) matchRepositories(ctx context.Context, rawTerm string, src *sources, sfidToClaGroups map[string][]string, m *matcher) error { - path := repositoryPath(rawTerm) +// matchRepositories resolves the pre-fetched repositories of a pasted repository URL or "owner/repo" +// path to the CLA Group owning that repository, and matches its owner segment against the linked +// organizations - the only signal available for an auto-enabled organization, whose repositories +// carry no records +func (s *service) matchRepositories(ctx context.Context, path string, repos []*RepositoryRow, src *sources, sfidToClaGroups map[string][]string, m *matcher) error { if path == "" { return nil } @@ -240,14 +258,11 @@ func (s *service) matchRepositories(ctx context.Context, rawTerm string, src *so } } - repos, err := s.repo.GetRepositoriesByName(ctx, nameVariants(path)) - if err != nil { - return err - } // the repository-name-index GSI is keyed on the case-preserved name, so a lower-cased paste of a // mixed-case repository misses it - the owner's repositories are then listed through the // organization GSI and compared case-insensitively if len(repos) == 0 && len(ownerOrgs) > 0 { + var err error repos, err = s.repo.GetRepositoriesByOrganization(ctx, organizationNames(ownerOrgs)) if err != nil { return err From 9d25e12521ed6f3ab3eefe4013764ff8272e1712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Gryglicki?= Date: Wed, 19 Aug 2026 06:48:39 +0000 Subject: [PATCH 3/7] Search CLA group API - 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) Assisted by [Claude](https://claude.ai) --- cla-backend-go/swagger/cla.v2.yaml | 2 +- cla-backend-go/v2/cla_search/cache.go | 184 +++++++++++++++++++ cla-backend-go/v2/cla_search/cache_test.go | 151 +++++++++++++++ cla-backend-go/v2/cla_search/handlers.go | 10 +- cla-backend-go/v2/cla_search/repository.go | 42 +++-- cla-backend-go/v2/cla_search/service.go | 32 +++- cla-backend-go/v2/cla_search/service_test.go | 89 ++++++++- utils/cla_search.sh | 24 ++- 8 files changed, 506 insertions(+), 28 deletions(-) create mode 100644 cla-backend-go/v2/cla_search/cache.go create mode 100644 cla-backend-go/v2/cla_search/cache_test.go diff --git a/cla-backend-go/swagger/cla.v2.yaml b/cla-backend-go/swagger/cla.v2.yaml index 87c49bd09..1f9d76a7c 100644 --- a/cla-backend-go/swagger/cla.v2.yaml +++ b/cla-backend-go/swagger/cla.v2.yaml @@ -2865,7 +2865,7 @@ paths: /cla-group/search: get: summary: Search CLA Groups - description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422 + description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422; a searchTerm that is shorter than 3 characters only after whitespace trimming is rejected with a 400. The reference data is served from an in-process cache with a short TTL, so a newly added CLA Group, organization or project mapping can take a few minutes to become searchable operationId: searchClaGroups parameters: - $ref: "#/parameters/x-request-id" diff --git a/cla-backend-go/v2/cla_search/cache.go b/cla-backend-go/v2/cla_search/cache.go new file mode 100644 index 000000000..7d804497d --- /dev/null +++ b/cla-backend-go/v2/cla_search/cache.go @@ -0,0 +1,184 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package cla_search + +import ( + "context" + "os" + "sync" + "time" + + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/sirupsen/logrus" + "golang.org/x/sync/singleflight" +) + +// DefaultCacheTTL is how long a scanned table is served from memory before the next search re-scans it +const DefaultCacheTTL = 5 * time.Minute + +// cacheTTLEnvVar overrides DefaultCacheTTL with any duration Go can parse, "0" disabling the cache +const cacheTTLEnvVar = "CLA_SEARCH_CACHE_TTL" + +func cacheTTL() time.Duration { + value := os.Getenv(cacheTTLEnvVar) + if value == "" { + return DefaultCacheTTL + } + ttl, err := time.ParseDuration(value) + if err != nil || ttl < 0 { + log.WithField(cacheTTLEnvVar, value).Warn("unable to parse the CLA Group search cache TTL - using the default") + return DefaultCacheTTL + } + return ttl +} + +// tableCache holds the rows of one scanned table until they age out. The rows are shared with every +// search that reads them and must be treated as read-only. +type tableCache[T any] struct { + name string + ttl time.Duration + load func(context.Context) ([]T, error) + flight singleflight.Group + + mu sync.RWMutex + rows []T + loadedAt time.Time +} + +func newTableCache[T any](name string, ttl time.Duration, load func(context.Context) ([]T, error)) *tableCache[T] { + return &tableCache[T]{name: name, ttl: ttl, load: load} +} + +// get returns the cached rows, scanning the table when they are absent or stale. Concurrent misses +// share a single scan. +func (c *tableCache[T]) get(ctx context.Context) ([]T, error) { + if rows, ok := c.fresh(); ok { + return rows, nil + } + + f := logrus.Fields{"functionName": "v2.cla_search.cache.get", "tableName": c.name} + _, err, shared := c.flight.Do(c.name, func() (interface{}, error) { + // a scan that finished while this call waited for the lock makes this one unnecessary + if _, ok := c.fresh(); ok { + return nil, nil + } + rows, loadErr := c.load(ctx) + if loadErr != nil { + return nil, loadErr + } + c.mu.Lock() + c.rows, c.loadedAt = rows, time.Now() + c.mu.Unlock() + log.WithFields(f).Debugf("cache miss - loaded %d rows", len(rows)) + return nil, nil + }) + if err != nil { + return nil, err + } + if shared { + log.WithFields(f).Debug("cache miss - served by a concurrent load") + } + c.mu.RLock() + defer c.mu.RUnlock() + return c.rows, nil +} + +func (c *tableCache[T]) fresh() ([]T, bool) { + c.mu.RLock() + rows, loadedAt := c.rows, c.loadedAt + c.mu.RUnlock() + if loadedAt.IsZero() || time.Since(loadedAt) >= c.ttl { + return nil, false + } + return rows, true +} + +// keyedCache holds one tableCache per lookup key, for a read whose cost justifies caching but whose +// result set depends on the key +type keyedCache[T any] struct { + ttl time.Duration + load func(context.Context, string) ([]T, error) + + mu sync.Mutex + entries map[string]*tableCache[T] +} + +func newKeyedCache[T any](ttl time.Duration, load func(context.Context, string) ([]T, error)) *keyedCache[T] { + return &keyedCache[T]{ttl: ttl, load: load, entries: map[string]*tableCache[T]{}} +} + +func (k *keyedCache[T]) get(ctx context.Context, key string) ([]T, error) { + k.mu.Lock() + entry, ok := k.entries[key] + if !ok { + entry = newTableCache(key, k.ttl, func(loadCtx context.Context) ([]T, error) { return k.load(loadCtx, key) }) + k.entries[key] = entry + } + k.mu.Unlock() + return entry.get(ctx) +} + +// cachedRepository serves the scanned tables from memory and passes the indexed repository lookups, +// whose keys have no useful cache locality, straight through +type cachedRepository struct { + Repository + claGroups *tableCache[*ClaGroupRow] + mappings *tableCache[*ProjectMappingRow] + github *tableCache[*OrgRow] + gitlab *tableCache[*OrgRow] + gerrit *tableCache[*OrgRow] + + // listing every repository of an organization is the one indexed read expensive enough to cache - + // a large organization runs to hundreds of rows, and a pasted URL under it repeats the same listing + orgRepositories *keyedCache[*RepositoryRow] +} + +func newCachedRepository(repo Repository, ttl time.Duration) Repository { + if ttl == 0 { + return repo + } + return &cachedRepository{ + Repository: repo, + orgRepositories: newKeyedCache(ttl, func(ctx context.Context, organizationName string) ([]*RepositoryRow, error) { + return repo.GetRepositoriesByOrganization(ctx, []string{organizationName}) + }), + claGroups: newTableCache("projects", ttl, repo.GetClaGroups), + mappings: newTableCache("projects-cla-groups", ttl, repo.GetProjectMappings), + github: newTableCache("github-orgs", ttl, repo.GetGithubOrgs), + gitlab: newTableCache("gitlab-orgs", ttl, repo.GetGitlabOrgs), + gerrit: newTableCache("gerrit-instances", ttl, repo.GetGerritInstances), + } +} + +func (c *cachedRepository) GetClaGroups(ctx context.Context) ([]*ClaGroupRow, error) { + return c.claGroups.get(ctx) +} + +func (c *cachedRepository) GetProjectMappings(ctx context.Context) ([]*ProjectMappingRow, error) { + return c.mappings.get(ctx) +} + +func (c *cachedRepository) GetGithubOrgs(ctx context.Context) ([]*OrgRow, error) { + return c.github.get(ctx) +} + +func (c *cachedRepository) GetGitlabOrgs(ctx context.Context) ([]*OrgRow, error) { + return c.gitlab.get(ctx) +} + +func (c *cachedRepository) GetGerritInstances(ctx context.Context) ([]*OrgRow, error) { + return c.gerrit.get(ctx) +} + +func (c *cachedRepository) GetRepositoriesByOrganization(ctx context.Context, organizationNames []string) ([]*RepositoryRow, error) { + var rows []*RepositoryRow + for _, organizationName := range organizationNames { + cached, err := c.orgRepositories.get(ctx, organizationName) + if err != nil { + return nil, err + } + rows = append(rows, cached...) + } + return rows, nil +} diff --git a/cla-backend-go/v2/cla_search/cache_test.go b/cla-backend-go/v2/cla_search/cache_test.go new file mode 100644 index 000000000..c255c64d1 --- /dev/null +++ b/cla-backend-go/v2/cla_search/cache_test.go @@ -0,0 +1,151 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package cla_search + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func countOf(sequence []string, name string) int { + count := 0 + for _, entry := range sequence { + if entry == name { + count++ + } + } + return count +} + +func TestCacheScansEachTableOnceForRepeatedSearches(t *testing.T) { + repo := sampleRepo() + svc := NewService(newCachedRepository(repo, time.Minute)) + for i := 0; i < 5; i++ { + list, err := svc.Search(context.Background(), "kubernetes cla", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-kube"}, ids(list)) + } + for _, table := range []string{"claGroups", "mappings", "github", "gitlab", "gerrit"} { + assert.Equal(t, 1, countOf(repo.callSequence, table), table) + } +} + +func TestCacheLeavesTheIndexedRepositoryLookupsUncached(t *testing.T) { + repo := sampleRepo() + svc := NewService(newCachedRepository(repo, time.Minute)) + for i := 0; i < 3; i++ { + _, err := svc.Search(context.Background(), "OpenTimelineIO/OpenTimelineIO-Java-Bindings", 0) + require.NoError(t, err) + } + assert.Equal(t, 3, repo.repoCalls) +} + +func TestCacheReusesTheOrganizationRepositoryListing(t *testing.T) { + repo := sampleRepo() + svc := NewService(newCachedRepository(repo, time.Minute)) + for i := 0; i < 3; i++ { + list, err := svc.Search(context.Background(), "https://github.com/opentimelineio/opentimelineio-java-bindings", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-otio"}, ids(list)) + require.Equal(t, "OpenTimelineIO/OpenTimelineIO-Java-Bindings", list.Results[0].MatchedRepositoryName) + } + assert.Equal(t, [][]string{{"OpenTimelineIO"}}, repo.orgQueries) +} + +func TestCacheRescansWhenTheEntryIsStale(t *testing.T) { + var loads int32 + cache := newTableCache("projects", 20*time.Millisecond, func(_ context.Context) ([]*ClaGroupRow, error) { + atomic.AddInt32(&loads, 1) + return []*ClaGroupRow{{ClaGroupID: "cg-1"}}, nil + }) + + rows, err := cache.get(context.Background()) + require.NoError(t, err) + require.Len(t, rows, 1) + + _, err = cache.get(context.Background()) + require.NoError(t, err) + assert.Equal(t, int32(1), atomic.LoadInt32(&loads)) + + time.Sleep(30 * time.Millisecond) + _, err = cache.get(context.Background()) + require.NoError(t, err) + assert.Equal(t, int32(2), atomic.LoadInt32(&loads)) +} + +func TestCacheConcurrentMissesShareASingleScan(t *testing.T) { + var loads int32 + cache := newTableCache("projects", time.Minute, func(_ context.Context) ([]*ClaGroupRow, error) { + atomic.AddInt32(&loads, 1) + time.Sleep(50 * time.Millisecond) + return []*ClaGroupRow{{ClaGroupID: "cg-1"}}, nil + }) + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + rows, err := cache.get(context.Background()) + assert.NoError(t, err) + assert.Len(t, rows, 1) + }() + } + wg.Wait() + assert.Equal(t, int32(1), atomic.LoadInt32(&loads)) +} + +func TestCacheDoesNotRetainAFailedScan(t *testing.T) { + var loads int32 + cache := newTableCache("projects", time.Minute, func(_ context.Context) ([]*ClaGroupRow, error) { + if atomic.AddInt32(&loads, 1) == 1 { + return nil, errors.New("boom") + } + return []*ClaGroupRow{{ClaGroupID: "cg-1"}}, nil + }) + + _, err := cache.get(context.Background()) + require.Error(t, err) + + rows, err := cache.get(context.Background()) + require.NoError(t, err) + assert.Len(t, rows, 1) +} + +func TestCacheDisabledByAZeroTTL(t *testing.T) { + repo := sampleRepo() + assert.Equal(t, Repository(repo), newCachedRepository(repo, 0)) + + svc := NewService(newCachedRepository(repo, 0)) + for i := 0; i < 2; i++ { + _, err := svc.Search(context.Background(), "kubernetes", 0) + require.NoError(t, err) + } + assert.Equal(t, 2, countOf(repo.callSequence, "claGroups")) +} + +func TestCacheTTLFromTheEnvironment(t *testing.T) { + for _, tc := range []struct { + value string + expected time.Duration + }{ + {"", DefaultCacheTTL}, + {"90s", 90 * time.Second}, + {"0", 0}, + {"-1m", DefaultCacheTTL}, + {"not-a-duration", DefaultCacheTTL}, + } { + t.Run(tc.value, func(t *testing.T) { + t.Setenv(cacheTTLEnvVar, tc.value) + assert.Equal(t, tc.expected, cacheTTL()) + }) + } +} diff --git a/cla-backend-go/v2/cla_search/handlers.go b/cla-backend-go/v2/cla_search/handlers.go index 3583562b8..fe53fd82c 100644 --- a/cla-backend-go/v2/cla_search/handlers.go +++ b/cla-backend-go/v2/cla_search/handlers.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/LF-Engineering/lfx-kit/auth" "github.com/go-openapi/runtime/middleware" @@ -19,6 +20,10 @@ import ( const missingUsernameMsg = "the authenticated principal carries no username - unable to search" +// searchTimeout bounds a search well inside the API Gateway limit, so a stuck DynamoDB call fails +// the request rather than holding the Lambda open +const searchTimeout = 15 * time.Second + // authorized accepts a principal carrying a username, or an admin principal such as a machine token func authorized(authUser *auth.User) bool { return authUser != nil && (authUser.UserName != "" || utils.IsUserAdmin(authUser)) @@ -51,7 +56,10 @@ func Configure(api *operations.EasyclaAPI, service Service) { return claSearchOps.NewSearchClaGroupsBadRequest().WithXRequestID(reqID).WithPayload(utils.ErrorResponseBadRequest(reqID, msg)) } - result, err := service.Search(ctx, params.SearchTerm, utils.Int64Value(params.Limit)) + searchCtx, cancel := context.WithTimeout(ctx, searchTimeout) + defer cancel() + + result, err := service.Search(searchCtx, params.SearchTerm, utils.Int64Value(params.Limit)) if err != nil { msg := "unable to search the CLA Groups for the provided search term" log.WithFields(f).WithError(err).Warn(msg) diff --git a/cla-backend-go/v2/cla_search/repository.go b/cla-backend-go/v2/cla_search/repository.go index e60e8780f..378dd016b 100644 --- a/cla-backend-go/v2/cla_search/repository.go +++ b/cla-backend-go/v2/cla_search/repository.go @@ -33,13 +33,14 @@ const ( searchConcurrency = claGroupScanSegments + projectMappingScanSegments + 3*orgScanSegments + 2 ) -// ClaGroupRow is a CLA Group record, projected to the fields the search results carry +// ClaGroupRow is a CLA Group record, projected to the fields the search results carry. The CLA type +// flags are pointers because a missing attribute means true - the Pynamo default the v1 reader keeps. type ClaGroupRow struct { ClaGroupID string `dynamodbav:"project_id"` Name string `dynamodbav:"project_name"` ExternalID string `dynamodbav:"project_external_id"` - IclaEnabled bool `dynamodbav:"project_icla_enabled"` - CclaEnabled bool `dynamodbav:"project_ccla_enabled"` + IclaEnabled *bool `dynamodbav:"project_icla_enabled"` + CclaEnabled *bool `dynamodbav:"project_ccla_enabled"` } // ProjectMappingRow is a projects-cla-groups mapping record, projected to the fields the search results carry @@ -54,11 +55,12 @@ type ProjectMappingRow struct { // OrgRow is a repository-hosting organization - a GitHub organization, a GitLab group or a Gerrit instance type OrgRow struct { - Name string - URL string - Source string - ProjectSFID string - ClaGroupID string + Name string + URL string + Source string + ProjectSFID string + ClaGroupID string + AutoEnabledClaGroupID string } // RepositoryRow is the repository a pasted URL or "owner/repo" path resolved to @@ -90,8 +92,13 @@ type repository struct { repositoryTableName string } -// NewRepository creates a new instance of the CLA Group search repository +// NewRepository creates a new instance of the CLA Group search repository, with the scanned tables +// served from the in-process cache func NewRepository(awsSession *session.Session, stage string) Repository { + return newCachedRepository(newScanRepository(awsSession, stage), cacheTTL()) +} + +func newScanRepository(awsSession *session.Session, stage string) Repository { // a search fans out to more concurrent DynamoDB calls than the default two idle connections // per host can serve, which would leave most of them paying for a fresh TLS handshake transport := &http.Transport{ @@ -134,9 +141,10 @@ func (repo repository) GetProjectMappings(ctx context.Context) ([]*ProjectMappin } type orgDBRow struct { - Name string `dynamodbav:"organization_name"` - URL string `dynamodbav:"organization_url"` - ProjectSFID string `dynamodbav:"project_sfid"` + Name string `dynamodbav:"organization_name"` + URL string `dynamodbav:"organization_url"` + ProjectSFID string `dynamodbav:"project_sfid"` + AutoEnabledClaGroupID string `dynamodbav:"auto_enabled_cla_group_id"` } func (repo repository) GetGithubOrgs(ctx context.Context) ([]*OrgRow, error) { @@ -150,12 +158,14 @@ func (repo repository) GetGitlabOrgs(ctx context.Context) ([]*OrgRow, error) { func (repo repository) scanOrgs(ctx context.Context, tableName, source string) ([]*OrgRow, error) { var rows []*orgDBRow filter := enabledFilter() - if err := repo.scan(ctx, tableName, orgScanSegments, &filter, []string{"organization_name", "organization_url", "project_sfid"}, &rows); err != nil { + if err := repo.scan(ctx, tableName, orgScanSegments, &filter, + []string{"organization_name", "organization_url", "project_sfid", "auto_enabled_cla_group_id"}, &rows); err != nil { return nil, err } orgs := make([]*OrgRow, 0, len(rows)) for _, row := range rows { - orgs = append(orgs, &OrgRow{Name: row.Name, URL: row.URL, Source: source, ProjectSFID: row.ProjectSFID}) + orgs = append(orgs, &OrgRow{Name: row.Name, URL: row.URL, Source: source, ProjectSFID: row.ProjectSFID, + AutoEnabledClaGroupID: row.AutoEnabledClaGroupID}) } return orgs, nil } @@ -271,6 +281,10 @@ func (repo repository) scan(ctx context.Context, tableName string, segments int, "segments": segments, } + if len(attributes) == 0 { + return fmt.Errorf("no attributes to project from table %s", tableName) + } + names := make([]expression.NameBuilder, 0, len(attributes)) for _, attribute := range attributes { names = append(names, expression.Name(attribute)) diff --git a/cla-backend-go/v2/cla_search/service.go b/cla-backend-go/v2/cla_search/service.go index 31283b05f..ad69c7f18 100644 --- a/cla-backend-go/v2/cla_search/service.go +++ b/cla-backend-go/v2/cla_search/service.go @@ -302,12 +302,22 @@ func organizationNames(orgs []*OrgRow) []string { } // claGroupIDs returns the CLA Groups the organization is linked to - Gerrit instances reference the -// CLA Group directly, GitHub organizations and GitLab groups reference it by project SFID +// CLA Group directly, while a GitHub organization or GitLab group references it by project SFID, by +// the CLA Group its new repositories are auto-enabled into, or by both func (o *OrgRow) claGroupIDs(sfidToClaGroups map[string][]string) []string { if o.ClaGroupID != "" { return []string{o.ClaGroupID} } - return sfidToClaGroups[o.ProjectSFID] + mapped := sfidToClaGroups[o.ProjectSFID] + if o.AutoEnabledClaGroupID == "" { + return mapped + } + for _, claGroupID := range mapped { + if claGroupID == o.AutoEnabledClaGroupID { + return mapped + } + } + return append(append(make([]string, 0, len(mapped)+1), mapped...), o.AutoEnabledClaGroupID) } func indexProjectSFIDs(mappings []*ProjectMappingRow) map[string][]string { @@ -350,7 +360,13 @@ func buildList(searchTerm string, limit int64, matches map[string]*match, src *s results := make([]models.ClaSearchResult, 0, len(matches)) for claGroupID, m := range matches { - results = append(results, buildResult(claGroupID, m, claGroupByID[claGroupID], mappingsByClaGroup[claGroupID], orgsByClaGroup[claGroupID])) + result := buildResult(claGroupID, m, claGroupByID[claGroupID], mappingsByClaGroup[claGroupID], orgsByClaGroup[claGroupID]) + // a CLA Group with neither a record nor a mapping - a deleted one still referenced by an + // organization - has nothing to display + if result.ClaGroupName == "" && result.ProjectName == "" { + continue + } + results = append(results, result) } sort.Slice(results, func(i, j int) bool { if ri, rj := matches[results[i].ClaGroupID].rank, matches[results[j].ClaGroupID].rank; ri != rj { @@ -386,8 +402,8 @@ func buildResult(claGroupID string, m *match, claGroup *ClaGroupRow, mappings [] if claGroup != nil { result.ClaGroupName = claGroup.Name result.ProjectExternalID = claGroup.ExternalID - result.IclaEnabled = claGroup.IclaEnabled - result.CclaEnabled = claGroup.CclaEnabled + result.IclaEnabled = enabledOrDefault(claGroup.IclaEnabled) + result.CclaEnabled = enabledOrDefault(claGroup.CclaEnabled) } // A foundation-level CLA Group is marked by a mapping whose ProjectSFID equals its @@ -412,6 +428,12 @@ func buildResult(claGroupID string, m *match, claGroup *ClaGroupRow, mappings [] return result } +// enabledOrDefault reads a CLA type flag, a missing attribute meaning enabled - the Pynamo +// default=True the v1 CLA Group reader also honours +func enabledOrDefault(enabled *bool) bool { + return enabled == nil || *enabled +} + func displayName(result models.ClaSearchResult) string { if result.ProjectName != "" { return strings.ToLower(result.ProjectName) diff --git a/cla-backend-go/v2/cla_search/service_test.go b/cla-backend-go/v2/cla_search/service_test.go index ef152d1d3..3c28bc163 100644 --- a/cla-backend-go/v2/cla_search/service_test.go +++ b/cla-backend-go/v2/cla_search/service_test.go @@ -114,9 +114,9 @@ func (f *fakeRepo) GetRepositoriesByOrganization(_ context.Context, organization func sampleRepo() *fakeRepo { return &fakeRepo{ claGroups: []*ClaGroupRow{ - {ClaGroupID: "cg-kube", Name: "Kubernetes CLA", ExternalID: "a09-kube", IclaEnabled: true, CclaEnabled: true}, - {ClaGroupID: "cg-otio", Name: "OpenTimelineIO CLA", ExternalID: "a09-otio", CclaEnabled: true}, - {ClaGroupID: "cg-onap", Name: "ONAP CLA", ExternalID: "a09-onap-f", IclaEnabled: true}, + {ClaGroupID: "cg-kube", Name: "Kubernetes CLA", ExternalID: "a09-kube", IclaEnabled: flag(true), CclaEnabled: flag(true)}, + {ClaGroupID: "cg-otio", Name: "OpenTimelineIO CLA", ExternalID: "a09-otio", IclaEnabled: flag(false), CclaEnabled: flag(true)}, + {ClaGroupID: "cg-onap", Name: "ONAP CLA", ExternalID: "a09-onap-f", IclaEnabled: flag(true), CclaEnabled: flag(false)}, {ClaGroupID: "cg-orphan", Name: "Kubernetes Edge CLA"}, }, mappings: []*ProjectMappingRow{ @@ -150,6 +150,10 @@ func sampleRepo() *fakeRepo { } } +func flag(value bool) *bool { + return &value +} + func resultByID(list *models.ClaSearchList, claGroupID string) *models.ClaSearchResult { for i := range list.Results { if list.Results[i].ClaGroupID == claGroupID { @@ -427,6 +431,85 @@ func TestSearchRunsSourcesConcurrently(t *testing.T) { assert.Equal(t, 1, repo.repoCalls) } +func TestSearchMissingClaTypeFlagsDefaultToEnabled(t *testing.T) { + // a CLA Group row without the flag attributes is enabled for both types - the Pynamo default + list, err := NewService(sampleRepo()).Search(context.Background(), "kubernetes edge", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-orphan"}, ids(list)) + assert.True(t, list.Results[0].IclaEnabled) + assert.True(t, list.Results[0].CclaEnabled) + + otio := resultByID(mustSearch(t, "opentimelineio cla"), "cg-otio") + require.NotNil(t, otio) + assert.False(t, otio.IclaEnabled) + assert.True(t, otio.CclaEnabled) +} + +func mustSearch(t *testing.T, term string) *models.ClaSearchList { + t.Helper() + list, err := NewService(sampleRepo()).Search(context.Background(), term, 0) + require.NoError(t, err) + return list +} + +// autoEnabledRepo mirrors the production shape of an organization whose only link to a CLA Group is +// the group its new repositories are auto-enabled into +func autoEnabledRepo() *fakeRepo { + return &fakeRepo{ + claGroups: []*ClaGroupRow{ + {ClaGroupID: "cg-chips", Name: "CHIPS Alliance"}, + {ClaGroupID: "cg-mapped", Name: "Mapped CLA"}, + }, + mappings: []*ProjectMappingRow{ + {ClaGroupID: "cg-mapped", ClaGroupName: "Mapped CLA", ProjectSFID: "sfid-both", ProjectName: "Mapped Project"}, + }, + github: []*OrgRow{ + {Name: "chipsalliance", Source: sourceGitHub, ProjectSFID: "sfid-unmapped", AutoEnabledClaGroupID: "cg-chips"}, + {Name: "both-org", Source: sourceGitHub, ProjectSFID: "sfid-both", AutoEnabledClaGroupID: "cg-chips"}, + {Name: "same-org", Source: sourceGitHub, ProjectSFID: "sfid-both", AutoEnabledClaGroupID: "cg-mapped"}, + {Name: "blank-org", Source: sourceGitHub, ProjectSFID: "sfid-both", AutoEnabledClaGroupID: ""}, + {Name: "dangling-org", Source: sourceGitHub, AutoEnabledClaGroupID: "cg-deleted"}, + }, + } +} + +func TestSearchAutoEnabledOrgResolvesClaGroupWithoutAMapping(t *testing.T) { + list, err := NewService(autoEnabledRepo()).Search(context.Background(), "chipsalliance", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-chips"}, ids(list)) + assert.Equal(t, []string{matchOrganization}, list.Results[0].MatchTypes) + assert.Equal(t, "CHIPS Alliance", list.Results[0].ClaGroupName) +} + +func TestSearchAutoEnabledClaGroupUnionsWithTheMappedOnes(t *testing.T) { + list, err := NewService(autoEnabledRepo()).Search(context.Background(), "both-org", 0) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"cg-chips", "cg-mapped"}, ids(list)) +} + +func TestSearchAutoEnabledClaGroupIsNotDuplicated(t *testing.T) { + list, err := NewService(autoEnabledRepo()).Search(context.Background(), "same-org", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-mapped"}, ids(list)) + assert.Equal(t, []models.ClaSearchOrg{ + {Name: "blank-org", Source: sourceGitHub, URL: "https://github.com/blank-org"}, + {Name: "both-org", Source: sourceGitHub, URL: "https://github.com/both-org"}, + {Name: "same-org", Source: sourceGitHub, URL: "https://github.com/same-org"}, + }, list.Results[0].Organizations) +} + +func TestSearchBlankAutoEnabledClaGroupFallsBackToTheMapping(t *testing.T) { + list, err := NewService(autoEnabledRepo()).Search(context.Background(), "blank-org", 0) + require.NoError(t, err) + assert.Equal(t, []string{"cg-mapped"}, ids(list)) +} + +func TestSearchOmitsAClaGroupThatResolvesToNothingDisplayable(t *testing.T) { + list, err := NewService(autoEnabledRepo()).Search(context.Background(), "dangling-org", 0) + require.NoError(t, err) + assert.Empty(t, list.Results) +} + func TestRepositoryPath(t *testing.T) { for _, tc := range []struct { term string diff --git a/utils/cla_search.sh b/utils/cla_search.sh index bc78e832f..a1558d1bc 100755 --- a/utils/cla_search.sh +++ b/utils/cla_search.sh @@ -64,14 +64,23 @@ then fi [ -z "$RUNS" ] && RUNS=1 +if ! printf '%s' "$RUNS" | grep -Eq '^[1-9][0-9]*$' +then + echo "$0: RUNS must be a positive integer, got '${RUNS}'" + exit 4 +fi + body="$(mktemp)" times="$(mktemp)" +trap 'rm -f "$body" "$times"' EXIT INT TERM +failed=0 for i in $(seq 1 "$RUNS") do timing="$(curl -sS -G -XGET "${auth[@]}" -H "Content-Type: application/json" "${args[@]}" -w '%{http_code} %{time_total}' -o "$body" "$URL")" code="${timing% *}" secs="${timing#* }" echo "$secs" >> "$times" + case "$code" in 2??) ;; *) failed=$((failed+1)) ;; esac if [ "$i" = "1" ] then if command -v jq >/dev/null 2>&1 @@ -87,9 +96,16 @@ done if [ "$RUNS" != "1" ] then - sort -n "$times" | awk -v runs="$RUNS" '{t[NR]=$1} END { - p50=t[int((NR+1)*0.50+0.5)]; p95=t[int((NR+1)*0.95+0.5)] - printf "runs=%d min=%.3fs p50=%.3fs p95=%.3fs max=%.3fs\n", runs, t[1], p50, p95, t[NR] + sort -n "$times" | awk -v runs="$RUNS" -v failed="$failed" '{t[NR]=$1} END { + i50=int((NR+1)*0.50+0.5); if (i50>NR) i50=NR; if (i50<1) i50=1 + i95=int((NR+1)*0.95+0.5); if (i95>NR) i95=NR; if (i95<1) i95=1 + printf "runs=%d min=%.3fs p50=%.3fs p95=%.3fs max=%.3fs", runs, t[1], t[i50], t[i95], t[NR] + if (failed+0 > 0) printf " NON-2XX=%d (timings above are not a valid measurement)", failed + printf "\n" }' + + if [ "$failed" != "0" ] + then + exit 5 + fi fi -rm -f "$body" "$times" From 2af628ef236730e65efb1489b59683c9c9e40723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Gryglicki?= Date: Wed, 19 Aug 2026 06:54:37 +0000 Subject: [PATCH 4/7] Search CLA group API - 4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) Assisted by [Claude](https://claude.ai) --- cla-backend-go/swagger/cla.v2.yaml | 2 +- cla-backend-go/v2/cla_search/cache.go | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cla-backend-go/swagger/cla.v2.yaml b/cla-backend-go/swagger/cla.v2.yaml index 1f9d76a7c..94fc208d2 100644 --- a/cla-backend-go/swagger/cla.v2.yaml +++ b/cla-backend-go/swagger/cla.v2.yaml @@ -2865,7 +2865,7 @@ paths: /cla-group/search: get: summary: Search CLA Groups - description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422; a searchTerm that is shorter than 3 characters only after whitespace trimming is rejected with a 400. The reference data is served from an in-process cache with a short TTL, so a newly added CLA Group, organization or project mapping can take a few minutes to become searchable + description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422; a searchTerm that is shorter than 3 characters only after whitespace trimming is rejected with a 400. The reference data is served from an in-process cache with a short TTL, so a newly added CLA Group, organization or project mapping can take up to the cache TTL (30 minutes) to become searchable operationId: searchClaGroups parameters: - $ref: "#/parameters/x-request-id" diff --git a/cla-backend-go/v2/cla_search/cache.go b/cla-backend-go/v2/cla_search/cache.go index 7d804497d..bbf9437d0 100644 --- a/cla-backend-go/v2/cla_search/cache.go +++ b/cla-backend-go/v2/cla_search/cache.go @@ -14,8 +14,11 @@ import ( "golang.org/x/sync/singleflight" ) -// DefaultCacheTTL is how long a scanned table is served from memory before the next search re-scans it -const DefaultCacheTTL = 5 * time.Minute +// DefaultCacheTTL is how long a scanned table is served from memory before the next search re-scans +// it. It is deliberately longer than a typical Lambda execution environment lives, so a container +// normally scans each table once and never again - the reference data changes on the timescale of +// project onboarding, not of searches. +const DefaultCacheTTL = 30 * time.Minute // cacheTTLEnvVar overrides DefaultCacheTTL with any duration Go can parse, "0" disabling the cache const cacheTTLEnvVar = "CLA_SEARCH_CACHE_TTL" From 020bbd4f12ed52dc603c2d664aceaf0e7f1d3d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Gryglicki?= Date: Wed, 19 Aug 2026 07:24:38 +0000 Subject: [PATCH 5/7] Search CLA group API - 5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) Assisted by [Claude](https://claude.ai) --- cla-backend-go/swagger/cla.v2.yaml | 2 +- cla-backend-go/v2/cla_search/cache.go | 33 ++++++++++----- cla-backend-go/v2/cla_search/cache_test.go | 47 ++++++++++++++++++++++ 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/cla-backend-go/swagger/cla.v2.yaml b/cla-backend-go/swagger/cla.v2.yaml index 94fc208d2..6927c83c1 100644 --- a/cla-backend-go/swagger/cla.v2.yaml +++ b/cla-backend-go/swagger/cla.v2.yaml @@ -2865,7 +2865,7 @@ paths: /cla-group/search: get: summary: Search CLA Groups - description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422; a searchTerm that is shorter than 3 characters only after whitespace trimming is rejected with a 400. The reference data is served from an in-process cache with a short TTL, so a newly added CLA Group, organization or project mapping can take up to the cache TTL (30 minutes) to become searchable + description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422; a searchTerm that is shorter than 3 characters only after whitespace trimming is rejected with a 400. The reference data is served from an in-process cache with a short TTL, so a newly added CLA Group, organization or project mapping can take up to the cache TTL (30 minutes by default) to become searchable operationId: searchClaGroups parameters: - $ref: "#/parameters/x-request-id" diff --git a/cla-backend-go/v2/cla_search/cache.go b/cla-backend-go/v2/cla_search/cache.go index bbf9437d0..5cdbb9600 100644 --- a/cla-backend-go/v2/cla_search/cache.go +++ b/cla-backend-go/v2/cla_search/cache.go @@ -20,6 +20,9 @@ import ( // project onboarding, not of searches. const DefaultCacheTTL = 30 * time.Minute +// loadTimeout bounds a table fill, which runs detached from the request that triggered it +const loadTimeout = 30 * time.Second + // cacheTTLEnvVar overrides DefaultCacheTTL with any duration Go can parse, "0" disabling the cache const cacheTTLEnvVar = "CLA_SEARCH_CACHE_TTL" @@ -61,12 +64,16 @@ func (c *tableCache[T]) get(ctx context.Context) ([]T, error) { } f := logrus.Fields{"functionName": "v2.cla_search.cache.get", "tableName": c.name} - _, err, shared := c.flight.Do(c.name, func() (interface{}, error) { + fill := c.flight.DoChan(c.name, func() (interface{}, error) { // a scan that finished while this call waited for the lock makes this one unnecessary if _, ok := c.fresh(); ok { return nil, nil } - rows, loadErr := c.load(ctx) + // the fill outlives the request that happened to trigger it - one caller giving up must not + // fail the fill for every other caller waiting on it + loadCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), loadTimeout) + defer cancel() + rows, loadErr := c.load(loadCtx) if loadErr != nil { return nil, loadErr } @@ -76,15 +83,21 @@ func (c *tableCache[T]) get(ctx context.Context) ([]T, error) { log.WithFields(f).Debugf("cache miss - loaded %d rows", len(rows)) return nil, nil }) - if err != nil { - return nil, err - } - if shared { - log.WithFields(f).Debug("cache miss - served by a concurrent load") + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-fill: + if result.Err != nil { + return nil, result.Err + } + if result.Shared { + log.WithFields(f).Debug("cache miss - served by a concurrent load") + } + c.mu.RLock() + defer c.mu.RUnlock() + return c.rows, nil } - c.mu.RLock() - defer c.mu.RUnlock() - return c.rows, nil } func (c *tableCache[T]) fresh() ([]T, bool) { diff --git a/cla-backend-go/v2/cla_search/cache_test.go b/cla-backend-go/v2/cla_search/cache_test.go index c255c64d1..7b8a03790 100644 --- a/cla-backend-go/v2/cla_search/cache_test.go +++ b/cla-backend-go/v2/cla_search/cache_test.go @@ -103,6 +103,53 @@ func TestCacheConcurrentMissesShareASingleScan(t *testing.T) { assert.Equal(t, int32(1), atomic.LoadInt32(&loads)) } +func TestCacheFillSurvivesTheCancellationOfTheRequestThatTriggeredIt(t *testing.T) { + var loads int32 + started := make(chan struct{}, 1) + release := make(chan struct{}) + cache := newTableCache("projects", time.Minute, func(ctx context.Context) ([]*ClaGroupRow, error) { + atomic.AddInt32(&loads, 1) + started <- struct{}{} + <-release + if err := ctx.Err(); err != nil { + return nil, err + } + return []*ClaGroupRow{{ClaGroupID: "cg-1"}}, nil + }) + + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + leaderErr := make(chan error, 1) + go func() { + _, err := cache.get(leaderCtx) + leaderErr <- err + }() + <-started + + type waiterResult struct { + rows []*ClaGroupRow + err error + } + waiter := make(chan waiterResult, 1) + go func() { + rows, err := cache.get(context.Background()) + waiter <- waiterResult{rows: rows, err: err} + }() + time.Sleep(50 * time.Millisecond) + + cancelLeader() + assert.ErrorIs(t, <-leaderErr, context.Canceled) + + close(release) + result := <-waiter + require.NoError(t, result.err) + require.Len(t, result.rows, 1) + + rows, err := cache.get(context.Background()) + require.NoError(t, err) + assert.Len(t, rows, 1) + assert.Equal(t, int32(1), atomic.LoadInt32(&loads)) +} + func TestCacheDoesNotRetainAFailedScan(t *testing.T) { var loads int32 cache := newTableCache("projects", time.Minute, func(_ context.Context) ([]*ClaGroupRow, error) { From e5c082dc06946a774c5754808eee32027a500515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Gryglicki?= Date: Wed, 19 Aug 2026 07:51:34 +0000 Subject: [PATCH 6/7] Search CLA group API - 6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) Assisted by [Claude](https://claude.ai) --- cla-backend-go/swagger/cla.v2.yaml | 2 +- cla-backend-go/v2/cla_search/service.go | 123 ++++++++++++++----- cla-backend-go/v2/cla_search/service_test.go | 93 ++++++++++++-- 3 files changed, 174 insertions(+), 44 deletions(-) diff --git a/cla-backend-go/swagger/cla.v2.yaml b/cla-backend-go/swagger/cla.v2.yaml index 6927c83c1..787e9d301 100644 --- a/cla-backend-go/swagger/cla.v2.yaml +++ b/cla-backend-go/swagger/cla.v2.yaml @@ -2865,7 +2865,7 @@ paths: /cla-group/search: get: summary: Search CLA Groups - description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422; a searchTerm that is shorter than 3 characters only after whitespace trimming is rejected with a 400. The reference data is served from an in-process cache with a short TTL, so a newly added CLA Group, organization or project mapping can take up to the cache TTL (30 minutes by default) to become searchable + description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. A URL that names a known repository resolves to that repository's CLA Group only - the owning organization is used as a fallback when no repository record matches - and a github.com or gitlab.com URL is matched against repositories and organizations of that forge only, while a bare "owner/repo" path matches either. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422; a searchTerm that is shorter than 3 characters only after whitespace trimming is rejected with a 400. The reference data is served from an in-process cache with a short TTL, so a newly added CLA Group, organization or project mapping can take up to the cache TTL (30 minutes by default) to become searchable operationId: searchClaGroups parameters: - $ref: "#/parameters/x-request-id" diff --git a/cla-backend-go/v2/cla_search/service.go b/cla-backend-go/v2/cla_search/service.go index ad69c7f18..aaf76f864 100644 --- a/cla-backend-go/v2/cla_search/service.go +++ b/cla-backend-go/v2/cla_search/service.go @@ -39,8 +39,13 @@ const ( ) // forgeHosts are the shared repository hosts whose hostname carries no CLA Group signal, so a -// pasted URL on one of them is matched by its path only -var forgeHosts = map[string]bool{"github.com": true, "www.github.com": true, "gitlab.com": true, "www.gitlab.com": true} +// pasted URL on one of them is matched by its path only, and the forge each one identifies +var forgeHosts = map[string]string{ + "github.com": sourceGitHub, + "www.github.com": sourceGitHub, + "gitlab.com": sourceGitLab, + "www.gitlab.com": sourceGitLab, +} // Service interface defines the CLA Group search service type Service interface { @@ -78,7 +83,7 @@ func (s *service) Search(ctx context.Context, searchTerm string, limit int64) (* src *sources repos []*RepositoryRow ) - path := repositoryPath(rawTerm) + path, forge := repositoryPath(rawTerm) fetch, fetchCtx := errgroup.WithContext(ctx) fetch.Go(func() error { var err error @@ -102,7 +107,7 @@ func (s *service) Search(ctx context.Context, searchTerm string, limit int64) (* searchers.Go(func() error { matchClaGroupNames(src.claGroups, term, m); return nil }) searchers.Go(func() error { matchProjectNames(src.mappings, term, m); return nil }) searchers.Go(func() error { matchOrgNames(src.orgs, term, sfidToClaGroups, m); return nil }) - searchers.Go(func() error { return s.matchRepositories(searchCtx, path, repos, src, sfidToClaGroups, m) }) + searchers.Go(func() error { return s.matchRepositories(searchCtx, path, forge, repos, src, sfidToClaGroups, m) }) if err := searchers.Wait(); err != nil { return nil, err } @@ -235,47 +240,103 @@ func orgRank(org *OrgRow, term, host string) int { if strings.Contains(urlSignal(org.URL), term) { return rankSubstring } - if host != "" && !forgeHosts[host] && hostOf(org.URL) == host { + if host != "" && forgeHosts[host] == "" && hostOf(org.URL) == host { return rankSubstring } return -1 } // matchRepositories resolves the pre-fetched repositories of a pasted repository URL or "owner/repo" -// path to the CLA Group owning that repository, and matches its owner segment against the linked -// organizations - the only signal available for an auto-enabled organization, whose repositories -// carry no records -func (s *service) matchRepositories(ctx context.Context, path string, repos []*RepositoryRow, src *sources, sfidToClaGroups map[string][]string, m *matcher) error { +// path to the CLA Group owning that repository. A pasted URL names exactly one repository, so its +// owner organization is only consulted when no repository record answers - the case of an +// auto-enabled organization, whose repositories carry no records +func (s *service) matchRepositories(ctx context.Context, path, forge string, repos []*RepositoryRow, src *sources, sfidToClaGroups map[string][]string, m *matcher) error { if path == "" { return nil } owner := path[:strings.Index(path, "/")] + ownerOrgs := orgsOnForge(orgsNamed(src.orgs, owner), forge) - ownerOrgs := orgsNamed(src.orgs, owner) - for _, org := range ownerOrgs { - for _, claGroupID := range org.claGroupIDs(sfidToClaGroups) { - m.record(claGroupID, matchOrganization, rankExact) - } - } - + matched := reposNamed(repos, path, forge) // the repository-name-index GSI is keyed on the case-preserved name, so a lower-cased paste of a // mixed-case repository misses it - the owner's repositories are then listed through the // organization GSI and compared case-insensitively - if len(repos) == 0 && len(ownerOrgs) > 0 { - var err error - repos, err = s.repo.GetRepositoriesByOrganization(ctx, organizationNames(ownerOrgs)) + if len(matched) == 0 && len(ownerOrgs) > 0 { + listed, err := s.repo.GetRepositoriesByOrganization(ctx, organizationNames(ownerOrgs)) if err != nil { return err } + matched = reposNamed(listed, path, forge) } + resolved := false + for _, repo := range matched { + if !displayableClaGroup(src, repo.ClaGroupID) { + continue + } + m.recordRepository(repo.ClaGroupID, repo.Name, repo.URL) + resolved = true + } + if resolved { + return nil + } + + for _, org := range ownerOrgs { + for _, claGroupID := range org.claGroupIDs(sfidToClaGroups) { + m.record(claGroupID, matchOrganization, rankExact) + } + } + return nil +} + +// reposNamed returns the repositories whose full name is the given path, restricted to the forge the +// term named - a bare "owner/repo" names none and matches either forge +func reposNamed(repos []*RepositoryRow, path, forge string) []*RepositoryRow { lowerPath := strings.ToLower(path) + var matched []*RepositoryRow for _, repo := range repos { - if strings.ToLower(repo.Name) == lowerPath { - m.recordRepository(repo.ClaGroupID, repo.Name, repo.URL) + if strings.ToLower(repo.Name) != lowerPath { + continue } + if forge != "" && repo.Type != "" && !strings.EqualFold(repo.Type, forge) { + continue + } + matched = append(matched, repo) } - return nil + return matched +} + +// orgsOnForge drops the organizations of another forge than the one the term named +func orgsOnForge(orgs []*OrgRow, forge string) []*OrgRow { + if forge == "" { + return orgs + } + matched := make([]*OrgRow, 0, len(orgs)) + for _, org := range orgs { + if org.Source == "" || strings.EqualFold(org.Source, forge) { + matched = append(matched, org) + } + } + return matched +} + +// displayableClaGroup reports whether the CLA Group has a record or a mapping to show - a repository +// pointing at a deleted CLA Group resolves to nothing and must not suppress the organization match +func displayableClaGroup(src *sources, claGroupID string) bool { + if claGroupID == "" { + return false + } + for _, claGroup := range src.claGroups { + if claGroup.ClaGroupID == claGroupID { + return true + } + } + for _, mapping := range src.mappings { + if mapping.ClaGroupID == claGroupID { + return true + } + } + return false } // orgsNamed returns the organizations whose name is the given name, compared case-insensitively @@ -499,24 +560,24 @@ func hostOf(rawURL string) string { } // repositoryPath derives the full repository name the term addresses - the path of a pasted -// repository URL, or the term itself when it looks like an "owner/repo" path - and is empty when -// the term addresses no repository -func repositoryPath(term string) string { - path := term +// repository URL, or the term itself when it looks like an "owner/repo" path - together with the +// forge the URL host names, and is empty when the term addresses no repository +func repositoryPath(term string) (string, string) { + path, forge := term, "" if strings.Contains(term, "://") { parsed, err := url.Parse(term) if err != nil || parsed.Hostname() == "" { - return "" + return "", "" } - path = parsed.Path + path, forge = parsed.Path, forgeHosts[strings.ToLower(parsed.Hostname())] } path = strings.Trim(path, "/") path = strings.TrimSuffix(path, ".git") path = strings.TrimPrefix(path, "groups/") if !strings.Contains(path, "/") || strings.ContainsAny(path, " \t") { - return "" + return "", "" } - return path + return path, forge } // nameVariants are the repository names looked up on the case-preserved repository-name-index GSI @@ -535,7 +596,7 @@ func urlSignal(rawURL string) string { if err != nil { return strings.ToLower(rawURL) } - if forgeHosts[parsed.Hostname()] { + if forgeHosts[parsed.Hostname()] != "" { return strings.Trim(parsed.Path, "/") } return parsed.Hostname() + parsed.Path diff --git a/cla-backend-go/v2/cla_search/service_test.go b/cla-backend-go/v2/cla_search/service_test.go index 3c28bc163..9c13ed013 100644 --- a/cla-backend-go/v2/cla_search/service_test.go +++ b/cla-backend-go/v2/cla_search/service_test.go @@ -248,7 +248,7 @@ func TestSearchResolvesPastedRepoURL(t *testing.T) { list, err := svc.Search(context.Background(), term, 0) require.NoError(t, err) require.Equal(t, []string{"cg-otio"}, ids(list)) - assert.Contains(t, list.Results[0].MatchTypes, matchRepository) + assert.Equal(t, []string{matchRepository}, list.Results[0].MatchTypes) assert.Equal(t, "OpenTimelineIO/OpenTimelineIO-Java-Bindings", list.Results[0].MatchedRepositoryName) assert.Equal(t, "https://github.com/OpenTimelineIO/OpenTimelineIO-Java-Bindings", list.Results[0].MatchedRepositoryURL) }) @@ -260,7 +260,7 @@ func TestSearchResolvesNestedGitlabRepoURL(t *testing.T) { list, err := svc.Search(context.Background(), "https://gitlab.com/onap/oom/oom", 0) require.NoError(t, err) require.Equal(t, []string{"cg-onap"}, ids(list)) - assert.Contains(t, list.Results[0].MatchTypes, matchRepository) + assert.Equal(t, []string{matchRepository}, list.Results[0].MatchTypes) } func TestSearchResolvesGerritHostURL(t *testing.T) { @@ -286,7 +286,7 @@ func TestSearchResolvesLowerCasedPasteOfMixedCaseRepoURL(t *testing.T) { list, err := svc.Search(context.Background(), "https://github.com/opentimelineio/opentimelineio-java-bindings", 0) require.NoError(t, err) require.Equal(t, []string{"cg-otio"}, ids(list)) - assert.Contains(t, list.Results[0].MatchTypes, matchRepository) + assert.Equal(t, []string{matchRepository}, list.Results[0].MatchTypes) assert.Equal(t, "OpenTimelineIO/OpenTimelineIO-Java-Bindings", list.Results[0].MatchedRepositoryName) assert.Equal(t, [][]string{{"OpenTimelineIO"}}, repo.orgQueries) } @@ -514,21 +514,90 @@ func TestRepositoryPath(t *testing.T) { for _, tc := range []struct { term string expected string + forge string variants []string }{ - {"kubernetes", "", nil}, - {"has space/repo", "", nil}, - {"not-a-url://", "", nil}, - {"Owner/Repo", "Owner/Repo", []string{"Owner/Repo", "owner/repo"}}, - {"owner/repo", "owner/repo", []string{"owner/repo"}}, - {"https://gitlab.com/groups/onap", "", nil}, - {"https://gitlab.com/onap/oom/oom", "onap/oom/oom", []string{"onap/oom/oom"}}, - {"https://github.com/Owner/Repo.git", "Owner/Repo", []string{"Owner/Repo", "owner/repo"}}, + {"kubernetes", "", "", nil}, + {"has space/repo", "", "", nil}, + {"not-a-url://", "", "", nil}, + {"Owner/Repo", "Owner/Repo", "", []string{"Owner/Repo", "owner/repo"}}, + {"owner/repo", "owner/repo", "", []string{"owner/repo"}}, + {"https://gitlab.com/groups/onap", "", "", nil}, + {"https://gitlab.com/onap/oom/oom", "onap/oom/oom", sourceGitLab, []string{"onap/oom/oom"}}, + {"https://github.com/Owner/Repo.git", "Owner/Repo", sourceGitHub, []string{"Owner/Repo", "owner/repo"}}, + {"https://WWW.GitHub.com/Owner/Repo", "Owner/Repo", sourceGitHub, []string{"Owner/Repo", "owner/repo"}}, + {"https://gerrit.onap.org/r/aai/aai-common", "r/aai/aai-common", "", []string{"r/aai/aai-common"}}, } { - path := repositoryPath(tc.term) + path, forge := repositoryPath(tc.term) assert.Equal(t, tc.expected, path, tc.term) + assert.Equal(t, tc.forge, forge, tc.term) if path != "" { assert.Equal(t, tc.variants, nameVariants(path), tc.term) } } } + +// divergentRepo mirrors the AcademySoftwareFoundation shape: an organization auto-enabled into one +// CLA Group while hosting a repository owned by another, and the same organization name on the other +// forge behind a different CLA Group +func divergentRepo() *fakeRepo { + return &fakeRepo{ + claGroups: []*ClaGroupRow{ + {ClaGroupID: "cg-org", Name: "MoonRay"}, + {ClaGroupID: "cg-repo", Name: "Dailies Notes Assistant"}, + {ClaGroupID: "cg-gitlab", Name: "GitLab Group CLA"}, + }, + github: []*OrgRow{{Name: "aswf", Source: sourceGitHub, AutoEnabledClaGroupID: "cg-org"}}, + gitlab: []*OrgRow{{Name: "aswf", Source: sourceGitLab, AutoEnabledClaGroupID: "cg-gitlab"}}, + repos: map[string][]*RepositoryRow{ + "aswf/dna": {{Name: "aswf/dna", URL: "https://github.com/aswf/dna", Type: sourceGitHub, ClaGroupID: "cg-repo"}}, + "aswf/only-on-gitlab": {{Name: "aswf/only-on-gitlab", URL: "https://gitlab.com/aswf/only-on-gitlab", Type: sourceGitLab, ClaGroupID: "cg-gitlab"}}, + "aswf/ghost": {{Name: "aswf/ghost", URL: "https://github.com/aswf/ghost", Type: sourceGitHub, ClaGroupID: "cg-deleted"}}, + }, + } +} + +func TestSearchPastedURLResolvesToTheRepositoryOwnerNotTheOrganization(t *testing.T) { + list, err := NewService(divergentRepo()).Search(context.Background(), "https://github.com/aswf/dna", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-repo"}, ids(list)) + assert.Equal(t, []string{matchRepository}, list.Results[0].MatchTypes) + assert.Equal(t, "aswf/dna", list.Results[0].MatchedRepositoryName) +} + +func TestSearchPastedURLFallsBackToTheOrganizationWhenTheRepositoryGroupIsGone(t *testing.T) { + list, err := NewService(divergentRepo()).Search(context.Background(), "https://github.com/aswf/ghost", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-org"}, ids(list)) + assert.Equal(t, []string{matchOrganization}, list.Results[0].MatchTypes) + assert.Empty(t, list.Results[0].MatchedRepositoryName) +} + +func TestSearchPastedURLIgnoresARepositoryOnAnotherForge(t *testing.T) { + list, err := NewService(divergentRepo()).Search(context.Background(), "https://github.com/aswf/only-on-gitlab", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-org"}, ids(list)) + assert.Equal(t, []string{matchOrganization}, list.Results[0].MatchTypes) + + list, err = NewService(divergentRepo()).Search(context.Background(), "https://gitlab.com/aswf/only-on-gitlab", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-gitlab"}, ids(list)) + assert.Equal(t, []string{matchRepository}, list.Results[0].MatchTypes) +} + +func TestSearchBareOwnerRepoMatchesEitherForge(t *testing.T) { + list, err := NewService(divergentRepo()).Search(context.Background(), "aswf/only-on-gitlab", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-gitlab"}, ids(list)) + assert.Equal(t, []string{matchRepository}, list.Results[0].MatchTypes) +} + +func TestSearchOrganizationFallbackStaysOnTheForgeTheURLNamed(t *testing.T) { + list, err := NewService(divergentRepo()).Search(context.Background(), "https://github.com/aswf/unknown-repo", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-org"}, ids(list)) + + list, err = NewService(divergentRepo()).Search(context.Background(), "https://gitlab.com/aswf/unknown-repo", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-gitlab"}, ids(list)) +} From cdcd9659205435d754045da83b6f3f912ec1e71f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Gryglicki?= Date: Wed, 19 Aug 2026 08:13:41 +0000 Subject: [PATCH 7/7] Search CLA group API - 7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) Assisted by [Claude](https://claude.ai) --- cla-backend-go/swagger/cla.v2.yaml | 2 +- cla-backend-go/v2/cla_search/service.go | 51 ++++++++++++-------- cla-backend-go/v2/cla_search/service_test.go | 39 ++++++++++++--- 3 files changed, 65 insertions(+), 27 deletions(-) diff --git a/cla-backend-go/swagger/cla.v2.yaml b/cla-backend-go/swagger/cla.v2.yaml index 787e9d301..6e9ce1302 100644 --- a/cla-backend-go/swagger/cla.v2.yaml +++ b/cla-backend-go/swagger/cla.v2.yaml @@ -2865,7 +2865,7 @@ paths: /cla-group/search: get: summary: Search CLA Groups - description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. A URL that names a known repository resolves to that repository's CLA Group only - the owning organization is used as a fallback when no repository record matches - and a github.com or gitlab.com URL is matched against repositories and organizations of that forge only, while a bare "owner/repo" path matches either. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422; a searchTerm that is shorter than 3 characters only after whitespace trimming is rejected with a 400. The reference data is served from an in-process cache with a short TTL, so a newly added CLA Group, organization or project mapping can take up to the cache TTL (30 minutes by default) to become searchable + description: Unscoped search over the CLA Group name, the Salesforce project (or foundation) name, the names of the linked GitHub organizations, GitLab groups and Gerrit instances, and the repository the search term resolves to - a pasted repository URL or a "owner/repo" path is resolved to the CLA Group owning that repository. A URL that names a known repository resolves to that repository's CLA Group only - the owning organization is used as a fallback when no repository record matches - and a URL is matched against the repositories and organizations of the host it names - the forge for a github.com or gitlab.com URL, the host itself for a self-hosted one - while a bare "owner/repo" path matches either forge. Matching is case-insensitive substring matching performed server-side, results are deduplicated by CLA Group and capped at limit. A searchTerm shorter than 3 characters, or a limit outside its bounds, is rejected with a 422; a searchTerm that is shorter than 3 characters only after whitespace trimming is rejected with a 400. The reference data is served from an in-process cache with a short TTL, so a newly added CLA Group, organization or project mapping can take up to the cache TTL (30 minutes by default) to become searchable operationId: searchClaGroups parameters: - $ref: "#/parameters/x-request-id" diff --git a/cla-backend-go/v2/cla_search/service.go b/cla-backend-go/v2/cla_search/service.go index aaf76f864..46f95cc6d 100644 --- a/cla-backend-go/v2/cla_search/service.go +++ b/cla-backend-go/v2/cla_search/service.go @@ -83,7 +83,7 @@ func (s *service) Search(ctx context.Context, searchTerm string, limit int64) (* src *sources repos []*RepositoryRow ) - path, forge := repositoryPath(rawTerm) + path, host := repositoryPath(rawTerm) fetch, fetchCtx := errgroup.WithContext(ctx) fetch.Go(func() error { var err error @@ -107,7 +107,7 @@ func (s *service) Search(ctx context.Context, searchTerm string, limit int64) (* searchers.Go(func() error { matchClaGroupNames(src.claGroups, term, m); return nil }) searchers.Go(func() error { matchProjectNames(src.mappings, term, m); return nil }) searchers.Go(func() error { matchOrgNames(src.orgs, term, sfidToClaGroups, m); return nil }) - searchers.Go(func() error { return s.matchRepositories(searchCtx, path, forge, repos, src, sfidToClaGroups, m) }) + searchers.Go(func() error { return s.matchRepositories(searchCtx, path, host, repos, src, sfidToClaGroups, m) }) if err := searchers.Wait(); err != nil { return nil, err } @@ -250,14 +250,15 @@ func orgRank(org *OrgRow, term, host string) int { // path to the CLA Group owning that repository. A pasted URL names exactly one repository, so its // owner organization is only consulted when no repository record answers - the case of an // auto-enabled organization, whose repositories carry no records -func (s *service) matchRepositories(ctx context.Context, path, forge string, repos []*RepositoryRow, src *sources, sfidToClaGroups map[string][]string, m *matcher) error { +func (s *service) matchRepositories(ctx context.Context, path, host string, repos []*RepositoryRow, src *sources, sfidToClaGroups map[string][]string, m *matcher) error { if path == "" { return nil } owner := path[:strings.Index(path, "/")] - ownerOrgs := orgsOnForge(orgsNamed(src.orgs, owner), forge) + forge := forgeHosts[host] + ownerOrgs := orgsOnHost(orgsNamed(src.orgs, owner), host, forge) - matched := reposNamed(repos, path, forge) + matched := reposNamed(repos, path, host, forge) // the repository-name-index GSI is keyed on the case-preserved name, so a lower-cased paste of a // mixed-case repository misses it - the owner's repositories are then listed through the // organization GSI and compared case-insensitively @@ -266,7 +267,7 @@ func (s *service) matchRepositories(ctx context.Context, path, forge string, rep if err != nil { return err } - matched = reposNamed(listed, path, forge) + matched = reposNamed(listed, path, host, forge) } resolved := false @@ -289,16 +290,22 @@ func (s *service) matchRepositories(ctx context.Context, path, forge string, rep return nil } -// reposNamed returns the repositories whose full name is the given path, restricted to the forge the -// term named - a bare "owner/repo" names none and matches either forge -func reposNamed(repos []*RepositoryRow, path, forge string) []*RepositoryRow { +// reposNamed returns the repositories whose full name is the given path, restricted to the forge a +// known host names, or to the host itself when the host is a self-hosted one - a bare "owner/repo" +// names no host and matches either forge +func reposNamed(repos []*RepositoryRow, path, host, forge string) []*RepositoryRow { lowerPath := strings.ToLower(path) var matched []*RepositoryRow for _, repo := range repos { if strings.ToLower(repo.Name) != lowerPath { continue } - if forge != "" && repo.Type != "" && !strings.EqualFold(repo.Type, forge) { + switch { + case forge != "": + if repo.Type != "" && !strings.EqualFold(repo.Type, forge) { + continue + } + case host != "" && hostOf(repo.URL) != host: continue } matched = append(matched, repo) @@ -306,16 +313,22 @@ func reposNamed(repos []*RepositoryRow, path, forge string) []*RepositoryRow { return matched } -// orgsOnForge drops the organizations of another forge than the one the term named -func orgsOnForge(orgs []*OrgRow, forge string) []*OrgRow { - if forge == "" { +// orgsOnHost drops the organizations the host of a pasted URL rules out - the ones of another forge +// when the host names one, the ones of another host when it does not +func orgsOnHost(orgs []*OrgRow, host, forge string) []*OrgRow { + if host == "" { return orgs } matched := make([]*OrgRow, 0, len(orgs)) for _, org := range orgs { - if org.Source == "" || strings.EqualFold(org.Source, forge) { - matched = append(matched, org) + if forge != "" { + if org.Source != "" && !strings.EqualFold(org.Source, forge) { + continue + } + } else if hostOf(orgURL(org)) != host { + continue } + matched = append(matched, org) } return matched } @@ -561,15 +574,15 @@ func hostOf(rawURL string) string { // repositoryPath derives the full repository name the term addresses - the path of a pasted // repository URL, or the term itself when it looks like an "owner/repo" path - together with the -// forge the URL host names, and is empty when the term addresses no repository +// host the URL names, and is empty when the term addresses no repository func repositoryPath(term string) (string, string) { - path, forge := term, "" + path, host := term, "" if strings.Contains(term, "://") { parsed, err := url.Parse(term) if err != nil || parsed.Hostname() == "" { return "", "" } - path, forge = parsed.Path, forgeHosts[strings.ToLower(parsed.Hostname())] + path, host = parsed.Path, strings.ToLower(parsed.Hostname()) } path = strings.Trim(path, "/") path = strings.TrimSuffix(path, ".git") @@ -577,7 +590,7 @@ func repositoryPath(term string) (string, string) { if !strings.Contains(path, "/") || strings.ContainsAny(path, " \t") { return "", "" } - return path, forge + return path, host } // nameVariants are the repository names looked up on the case-preserved repository-name-index GSI diff --git a/cla-backend-go/v2/cla_search/service_test.go b/cla-backend-go/v2/cla_search/service_test.go index 9c13ed013..64bbcc744 100644 --- a/cla-backend-go/v2/cla_search/service_test.go +++ b/cla-backend-go/v2/cla_search/service_test.go @@ -514,7 +514,7 @@ func TestRepositoryPath(t *testing.T) { for _, tc := range []struct { term string expected string - forge string + host string variants []string }{ {"kubernetes", "", "", nil}, @@ -523,14 +523,14 @@ func TestRepositoryPath(t *testing.T) { {"Owner/Repo", "Owner/Repo", "", []string{"Owner/Repo", "owner/repo"}}, {"owner/repo", "owner/repo", "", []string{"owner/repo"}}, {"https://gitlab.com/groups/onap", "", "", nil}, - {"https://gitlab.com/onap/oom/oom", "onap/oom/oom", sourceGitLab, []string{"onap/oom/oom"}}, - {"https://github.com/Owner/Repo.git", "Owner/Repo", sourceGitHub, []string{"Owner/Repo", "owner/repo"}}, - {"https://WWW.GitHub.com/Owner/Repo", "Owner/Repo", sourceGitHub, []string{"Owner/Repo", "owner/repo"}}, - {"https://gerrit.onap.org/r/aai/aai-common", "r/aai/aai-common", "", []string{"r/aai/aai-common"}}, + {"https://gitlab.com/onap/oom/oom", "onap/oom/oom", "gitlab.com", []string{"onap/oom/oom"}}, + {"https://github.com/Owner/Repo.git", "Owner/Repo", "github.com", []string{"Owner/Repo", "owner/repo"}}, + {"https://WWW.GitHub.com/Owner/Repo", "Owner/Repo", "www.github.com", []string{"Owner/Repo", "owner/repo"}}, + {"https://gerrit.onap.org/r/aai/aai-common", "r/aai/aai-common", "gerrit.onap.org", []string{"r/aai/aai-common"}}, } { - path, forge := repositoryPath(tc.term) + path, host := repositoryPath(tc.term) assert.Equal(t, tc.expected, path, tc.term) - assert.Equal(t, tc.forge, forge, tc.term) + assert.Equal(t, tc.host, host, tc.term) if path != "" { assert.Equal(t, tc.variants, nameVariants(path), tc.term) } @@ -553,6 +553,7 @@ func divergentRepo() *fakeRepo { "aswf/dna": {{Name: "aswf/dna", URL: "https://github.com/aswf/dna", Type: sourceGitHub, ClaGroupID: "cg-repo"}}, "aswf/only-on-gitlab": {{Name: "aswf/only-on-gitlab", URL: "https://gitlab.com/aswf/only-on-gitlab", Type: sourceGitLab, ClaGroupID: "cg-gitlab"}}, "aswf/ghost": {{Name: "aswf/ghost", URL: "https://github.com/aswf/ghost", Type: sourceGitHub, ClaGroupID: "cg-deleted"}}, + "aswf/self-hosted": {{Name: "aswf/self-hosted", URL: "https://git.aswf.example/aswf/self-hosted", Type: sourceGitHub, ClaGroupID: "cg-repo"}}, }, } } @@ -601,3 +602,27 @@ func TestSearchOrganizationFallbackStaysOnTheForgeTheURLNamed(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"cg-gitlab"}, ids(list)) } + +func TestSearchPastedURLOnAnUnknownHostDoesNotMatchAnotherHostsRepository(t *testing.T) { + list, err := NewService(divergentRepo()).Search(context.Background(), "https://unrelated.example/aswf/dna", 0) + require.NoError(t, err) + assert.Empty(t, list.Results) +} + +func TestSearchPastedURLOnAnUnknownHostResolvesTheRepositoryOfThatHost(t *testing.T) { + list, err := NewService(divergentRepo()).Search(context.Background(), "https://git.aswf.example/aswf/self-hosted", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-repo"}, ids(list)) + assert.Equal(t, []string{matchRepository}, list.Results[0].MatchTypes) + assert.Equal(t, "aswf/self-hosted", list.Results[0].MatchedRepositoryName) +} + +func TestSearchPastedURLOnAnUnknownHostFallsBackToTheOrganizationOfThatHost(t *testing.T) { + repo := divergentRepo() + repo.claGroups = append(repo.claGroups, &ClaGroupRow{ClaGroupID: "cg-gerrit", Name: "Self Hosted CLA"}) + repo.gerrit = []*OrgRow{{Name: "aswf", URL: "https://git.aswf.example", Source: sourceGerrit, ClaGroupID: "cg-gerrit"}} + list, err := NewService(repo).Search(context.Background(), "https://git.aswf.example/aswf/unknown-repo", 0) + require.NoError(t, err) + require.Equal(t, []string{"cg-gerrit"}, ids(list)) + assert.Equal(t, []string{matchOrganization}, list.Results[0].MatchTypes) +}