Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ Emojis for the following are chosen based on [gitmoji](https://gitmoji.dev/).
- Scribe-Data data contracts are served via Scribe-Server to tell the client applications how to process the data ([#18](https://github.com/scribe-org/Scribe-Server/issues/18), [#57](https://github.com/scribe-org/Scribe-Server/issues/57)).
- Data contracts were switched from JSON to YAML for easier maintenance ([#56](https://github.com/scribe-org/Scribe-Server/pull/56)).
- Statistics for available languages are now shown via the API ([#44](https://github.com/scribe-org/Scribe-Server/issues/44)).
- Language statistics include counts of prepositions and profanity, which are null for languages without them.
- An entry/landing page was set up for Scribe-Server ([#48](https://github.com/scribe-org/Scribe-Server/issues/48)), followed by a dedicated deployment and download page ([#52](https://github.com/scribe-org/Scribe-Server/pull/52)).
- A translation data retrieval endpoint with validation logic was implemented ([#58](https://github.com/scribe-org/Scribe-Server/issues/58), [#59](https://github.com/scribe-org/Scribe-Server/issues/59)).
- SQLite databases are available for download directly from the Scribe-Server UI ([#6](https://github.com/scribe-org/Scribe-Server/issues/6)).
Expand Down Expand Up @@ -61,6 +62,7 @@ Emojis for the following are chosen based on [gitmoji](https://gitmoji.dev/).
- The CI workflow was updated to select the Go version via `go-version-file` ([#8](https://github.com/scribe-org/Scribe-Server/issues/8)) and later improved further ([#30](https://github.com/scribe-org/Scribe-Server/issues/30)).
- Unit tests were added for the language validator functions ([#78](https://github.com/scribe-org/Scribe-Server/issues/78)).
- Unit tests were added for the YAML normalizer function ([#81](https://github.com/scribe-org/Scribe-Server/issues/81)).
- Unit tests were added for the available languages handler ([#84](https://github.com/scribe-org/Scribe-Server/issues/84)).

### ♻️ Code Refactoring

Expand Down
10 changes: 8 additions & 2 deletions api/handlers/language.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ import (

// MARK: Languages Endpoints

// Function variables allow mocking database queries in unit tests.
var (
getAvailableLanguages = database.GetAvailableLanguages
getLanguageDataTypes = database.GetLanguageDataTypes
)

// GetAvailableLanguages returns a list of all supported languages and their available data types.
//
// @Summary List all supported languages
Expand All @@ -35,7 +41,7 @@ import (
// @Failure 500 {object} models.ErrorResponse "Internal server error occurred while fetching languages"
// @Router /api/v1/languages [get]
func GetAvailableLanguages(c *gin.Context) {
languages, err := database.GetAvailableLanguages()
languages, err := getAvailableLanguages()
if err != nil {
log.Printf("Error fetching available languages: %v", err)
HandleError(c, http.StatusInternalServerError, constants.ErrorFetchingLanguages)
Expand All @@ -44,7 +50,7 @@ func GetAvailableLanguages(c *gin.Context) {

var languageInfos []models.LanguageInfo
for _, lang := range languages {
dataTypes, err := database.GetLanguageDataTypes(lang)
dataTypes, err := getLanguageDataTypes(lang)
if err != nil {
log.Printf("Error fetching data types for %s: %v", lang, err)
continue
Expand Down
157 changes: 157 additions & 0 deletions api/handlers/language_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
// SPDX-License-Identifier: GPL-3.0-or-later

package handlers

import (
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"testing"

"github.com/gin-gonic/gin"
"github.com/scribe-org/scribe-server/internal/constants"
"github.com/scribe-org/scribe-server/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// setupTestRouter creates a minimal Gin engine configured in test mode for testing the endpoint.
func setupTestRouter(t *testing.T) *gin.Engine {
t.Helper()

previousMode := gin.Mode()
gin.SetMode(gin.TestMode)
t.Cleanup(func() { gin.SetMode(previousMode) })

router := gin.New()
router.GET("/api/v1/languages", GetAvailableLanguages)
return router
}

// stubLanguageDatabase temporarily swaps package-level database function pointers for isolated unit testing,
func stubLanguageDatabase(
t *testing.T,
languagesFn func() ([]string, error),
dataTypesFn func(lang string) ([]string, error),
) {
t.Helper()

origLanguages := getAvailableLanguages
origDataTypes := getLanguageDataTypes

getAvailableLanguages = languagesFn
getLanguageDataTypes = dataTypesFn

t.Cleanup(func() {
getAvailableLanguages = origLanguages
getLanguageDataTypes = origDataTypes
})
}

// TestGetAvailableLanguages tests the GET /api/v1/languages handler under various database scenarios.
func TestGetAvailableLanguages(t *testing.T) {
router := setupTestRouter(t)

t.Run("200 with a populated array", func(t *testing.T) {
stubLanguageDatabase(
t,
func() ([]string, error) {
return []string{"en", "de"}, nil
},
func(lang string) ([]string, error) {
switch lang {
case "en":
return []string{"nouns", "verbs"}, nil
case "de":
return []string{"nouns", "verbs", "adjectives"}, nil
default:
return nil, fmt.Errorf("unexpected language: %s", lang)
}
},
)

req := httptest.NewRequest(http.MethodGet, "/api/v1/languages", nil)
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)

require.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, "application/json; charset=utf-8", rec.Header().Get("Content-Type"))

var response models.AvailableLanguagesResponse
require.NoError(t, json.NewDecoder(rec.Body).Decode(&response))

expected := models.AvailableLanguagesResponse{
Languages: []models.LanguageInfo{
{
Code: "en",
DataTypes: []string{"nouns", "verbs"},
},
{
Code: "de",
DataTypes: []string{"nouns", "verbs", "adjectives"},
},
},
}
assert.Equal(t, expected, response)
})

t.Run("500 when GetAvailableLanguages errors", func(t *testing.T) {
stubLanguageDatabase(
t,
func() ([]string, error) {
return nil, errors.New("database connection failed")
},
func(lang string) ([]string, error) {
return nil, nil
},
)

req := httptest.NewRequest(http.MethodGet, "/api/v1/languages", nil)
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)

require.Equal(t, http.StatusInternalServerError, rec.Code)
assert.Equal(t, "application/json; charset=utf-8", rec.Header().Get("Content-Type"))

var response models.ErrorResponse
require.NoError(t, json.NewDecoder(rec.Body).Decode(&response))
assert.Equal(t, constants.ErrorFetchingLanguages, response.Error)
})

t.Run("partial-failure when GetLanguageDataTypes fails for a language", func(t *testing.T) {
stubLanguageDatabase(
t,
func() ([]string, error) {
return []string{"en", "fr"}, nil
},
func(lang string) ([]string, error) {
if lang == "en" {
return []string{"nouns", "verbs"}, nil
}
return nil, errors.New("data types retrieval failed for fr")
},
)

req := httptest.NewRequest(http.MethodGet, "/api/v1/languages", nil)
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)

require.Equal(t, http.StatusOK, rec.Code)
assert.Equal(t, "application/json; charset=utf-8", rec.Header().Get("Content-Type"))

var response models.AvailableLanguagesResponse
require.NoError(t, json.NewDecoder(rec.Body).Decode(&response))

expected := models.AvailableLanguagesResponse{
Languages: []models.LanguageInfo{
{
Code: "en",
DataTypes: []string{"nouns", "verbs"},
},
},
}
assert.Equal(t, expected, response)
})
}
2 changes: 1 addition & 1 deletion api/handlers/statistics.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (

// GetLanguageStats handles GET /language-stats?codes=en,fr.
// @Summary Get statistics for one or multiple languages
// @Description Returns the number of nouns and verbs for the specified language codes.
// @Description Returns the number of nouns, verbs, prepositions and profanity entries for the given language codes (null if a language has none).
// @Tags statistics
// @Accept json
// @Produce json
Expand Down
89 changes: 70 additions & 19 deletions database/language.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"github.com/go-sql-driver/mysql"
"github.com/scribe-org/scribe-server/models"
"github.com/spf13/viper"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)

// MARK: Get Available Languages
Expand Down Expand Up @@ -89,32 +91,80 @@ func GetLanguageDataTypes(lang string) ([]string, error) {

// MARK: Language Statistics

// GetLanguageStat retrieves noun and verb statistics for a specific language.
// statDataTypes are the data types counted in language statistics in the order they're queried.
var statDataTypes = []string{"nouns", "verbs", "prepositions", "profanity"}

// requiredStatDataTypes are the data types a language needs to be included in statistics.
// Other data types such as prepositions are only available for some languages.
var requiredStatDataTypes = []string{"nouns", "verbs"}

// buildLanguageStatQuery builds a query that counts the entries of each statistics data type available for a language.
// It returns the query and the data types it counts in order, or no data types if a required data type is missing.
func buildLanguageStatQuery(lang string, availableDataTypes []string) (string, []string, error) {
available := make(map[string]bool, len(availableDataTypes))
for _, dataType := range availableDataTypes {
available[strings.ToLower(dataType)] = true
}

for _, dataType := range requiredStatDataTypes {
if !available[dataType] {
return "", nil, nil
}
}

caser := cases.Title(language.Und)
counts := make([]string, 0, len(statDataTypes))
countedDataTypes := make([]string, 0, len(statDataTypes))

for _, dataType := range statDataTypes {
if !available[dataType] {
continue
}

// Construct the table name format: ENLanguageDataNounsScribe.
tableName := strings.ToUpper(lang) + "LanguageData" + caser.String(dataType) + "Scribe"
if !IsValidTableName(tableName) {
return "", nil, fmt.Errorf("invalid table name for language: %s", lang)
}

counts = append(counts, fmt.Sprintf("(SELECT COUNT(*) FROM %s) AS %s", tableName, dataType))
countedDataTypes = append(countedDataTypes, dataType)
}

return "SELECT " + strings.Join(counts, ", "), countedDataTypes, nil
}

// GetLanguageStat retrieves statistics for the data types of a specific language.
func GetLanguageStat(lan string) (map[string]any, error) {
// Normalize and validate language code (e.g., "EN", "FR").
lang := strings.ToUpper(strings.TrimSpace(lan))
if !regexp.MustCompile(`^[A-Z]{2}$`).MatchString(lang) {
return nil, fmt.Errorf("invalid language code: %s", lang)
}

nounsTable := lang + "LanguageDataNounsScribe"
verbsTable := lang + "LanguageDataVerbsScribe"
dataTypes, err := GetLanguageDataTypes(lang)
if err != nil {
return nil, fmt.Errorf("error fetching data types for %s: %w", lang, err)
}

if !IsValidTableName(nounsTable) || !IsValidTableName(verbsTable) {
return nil, fmt.Errorf("invalid table names for language: %s", lang)
query, countedDataTypes, err := buildLanguageStatQuery(lang, dataTypes)
if err != nil {
return nil, err
}

query := fmt.Sprintf(`
SELECT
(SELECT COUNT(*) FROM %s) AS nouns,
(SELECT COUNT(*) FROM %s) AS verbs
`, nounsTable, verbsTable)
if len(countedDataTypes) == 0 {
log.Printf("⚠️ Skipping %s — missing nouns or verbs table", lang)
return nil, nil
}

// Execute the query.
row := DB.QueryRow(query)
counts := make([]int, len(countedDataTypes))
scanTargets := make([]any, len(counts))
for i := range counts {
scanTargets[i] = &counts[i]
}

var nouns, verbs int
err := row.Scan(&nouns, &verbs)
err = DB.QueryRow(query).Scan(scanTargets...)
if err != nil {
if mysqlErr, ok := err.(*mysql.MySQLError); ok && mysqlErr.Number == 1146 {
log.Printf("⚠️ Skipping %s — missing table: %s", lang, mysqlErr.Message)
Expand All @@ -128,14 +178,15 @@ func GetLanguageStat(lan string) (map[string]any, error) {
return nil, fmt.Errorf("error scanning stats for %s: %w", lang, err)
}

return map[string]any{
"code": strings.ToLower(lang),
"nouns": nouns,
"verbs": verbs,
}, nil
stat := map[string]any{"code": strings.ToLower(lang)}
for i, dataType := range countedDataTypes {
stat[dataType] = counts[i]
}

return stat, nil
}

// GetAllLanguageStats retrieves statistics for all available languages (only nouns and verbs).
// GetAllLanguageStats retrieves statistics for all available languages that have nouns and verbs.
func GetAllLanguageStats() ([]models.LanguageStatisticsReponse, error) {
availableLanguages, err := GetAvailableLanguages()
if err != nil {
Expand Down
Loading
Loading