Convert locale files from ini to json format (#35489)

Migrate from the current INI format to JSON for translations. JSON is
widely supported, including by platforms such as Crowdin and Weblate.

modules/translation/i18n/i18n.go

@@ -32,7 +32,7 @@ type LocaleStore interface {
 	// HasLang returns whether a given language is present in the store
 	HasLang(langName string) bool
 	// AddLocaleByIni adds a new language to the store
-	AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error
+	AddLocaleByJSON(langName, langDesc string, source, moreSource []byte) error
 }
 
 // ResetDefaultLocales resets the current default locales

modules/translation/i18n/i18n_test.go

@@ -13,24 +13,23 @@ import (
 
 func TestLocaleStore(t *testing.T) {
 	testData1 := []byte(`
-.dot.name = Dot Name
-fmt = %[1]s %[2]s
+{
+".dot.name": "Dot Name",
+"fmt": "%[1]s %[2]s",
 
-[section]
-sub = Sub String
-mixed = test value; <span style="color: red\; background: none;">%s</span>
-`)
+"section.sub": "Sub String",
+"section.mixed": "test value; <span style=\"color: red; background: none;\">%s</span>"
+}`)
 
 	testData2 := []byte(`
-fmt = %[2]s %[1]s
-
-[section]
-sub = Changed Sub String
-`)
+{
+	"fmt": "%[2]s %[1]s",
+	"section.sub": "Changed Sub String"
+}`)
 
 	ls := NewLocaleStore()
-	assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, nil))
-	assert.NoError(t, ls.AddLocaleByIni("lang2", "Lang2", testData2, nil))
+	assert.NoError(t, ls.AddLocaleByJSON("lang1", "Lang1", testData1, nil))
+	assert.NoError(t, ls.AddLocaleByJSON("lang2", "Lang2", testData2, nil))
 	ls.SetDefaultLang("lang1")
 
 	lang1, _ := ls.Locale("lang1")
@@ -69,17 +68,21 @@ sub = Changed Sub String
 
 func TestLocaleStoreMoreSource(t *testing.T) {
 	testData1 := []byte(`
-a=11
-b=12
+{
+	"a": "11",
+	"b": "12"
+}
 `)
 
 	testData2 := []byte(`
-b=21
-c=22
+{
+	"b": "21",
+	"c": "22"
+}
 `)
 
 	ls := NewLocaleStore()
-	assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", testData1, testData2))
+	assert.NoError(t, ls.AddLocaleByJSON("lang1", "Lang1", testData1, testData2))
 	lang1, _ := ls.Locale("lang1")
 	assert.Equal(t, "11", lang1.TrString("a"))
 	assert.Equal(t, "21", lang1.TrString("b"))
@@ -120,7 +123,7 @@ func (e *errorPointerReceiver) Error() string {
 
 func TestLocaleWithTemplate(t *testing.T) {
 	ls := NewLocaleStore()
-	assert.NoError(t, ls.AddLocaleByIni("lang1", "Lang1", []byte(`key=<a>%s</a>`), nil))
+	assert.NoError(t, ls.AddLocaleByJSON("lang1", "Lang1", []byte(`{"key":"<a>%s</a>"}`), nil))
 	lang1, _ := ls.Locale("lang1")
 
 	tmpl := template.New("test").Funcs(template.FuncMap{"tr": lang1.TrHTML})
@@ -150,57 +153,3 @@ func TestLocaleWithTemplate(t *testing.T) {
 		assert.Equal(t, c.want, buf.String())
 	}
 }
-
-func TestLocaleStoreQuirks(t *testing.T) {
-	const nl = "\n"
-	q := func(q1, s string, q2 ...string) string {
-		return q1 + s + strings.Join(q2, "")
-	}
-	testDataList := []struct {
-		in   string
-		out  string
-		hint string
-	}{
-		{` xx`, `xx`, "simple, no quote"},
-		{`" xx"`, ` xx`, "simple, double-quote"},
-		{`' xx'`, ` xx`, "simple, single-quote"},
-		{"` xx`", ` xx`, "simple, back-quote"},
-
-		{`x\"y`, `x\"y`, "no unescape, simple"},
-		{q(`"`, `x\"y`, `"`), `"x\"y"`, "unescape, double-quote"},
-		{q(`'`, `x\"y`, `'`), `x\"y`, "no unescape, single-quote"},
-		{q("`", `x\"y`, "`"), `x\"y`, "no unescape, back-quote"},
-
-		{q(`"`, `x\"y`) + nl + "b=", `"x\"y`, "half open, double-quote"},
-		{q(`'`, `x\"y`) + nl + "b=", `'x\"y`, "half open, single-quote"},
-		{q("`", `x\"y`) + nl + "b=`", `x\"y` + nl + "b=", "half open, back-quote, multi-line"},
-
-		{`x ; y`, `x ; y`, "inline comment (;)"},
-		{`x # y`, `x # y`, "inline comment (#)"},
-		{`x \; y`, `x ; y`, `inline comment (\;)`},
-		{`x \# y`, `x # y`, `inline comment (\#)`},
-	}
-
-	for _, testData := range testDataList {
-		ls := NewLocaleStore()
-		err := ls.AddLocaleByIni("lang1", "Lang1", []byte("a="+testData.in), nil)
-		lang1, _ := ls.Locale("lang1")
-		assert.NoError(t, err, testData.hint)
-		assert.Equal(t, testData.out, lang1.TrString("a"), testData.hint)
-		assert.NoError(t, ls.Close())
-	}
-
-	// TODO: Crowdin needs the strings to be quoted correctly and doesn't like incomplete quotes
-	//       and Crowdin always outputs quoted strings if there are quotes in the strings.
-	//       So, Gitea's `key="quoted" unquoted` content shouldn't be used on Crowdin directly,
-	//       it should be converted to `key="\"quoted\" unquoted"` first.
-	// TODO: We can not use UnescapeValueDoubleQuotes=true, because there are a lot of back-quotes in en-US.ini,
-	//       then Crowdin will output:
-	//       > key = "`x \" y`"
-	//       Then Gitea will read a string with back-quotes, which is incorrect.
-	// TODO: Crowdin might generate multi-line strings, quoted by double-quote, it's not supported by LocaleStore
-	//       LocaleStore uses back-quote for multi-line strings, it's not supported by Crowdin.
-	// TODO: Crowdin doesn't support back-quote as string quoter, it mainly uses double-quote
-	//       so, the following line will be parsed as: value="`first", comment="second`" on Crowdin
-	//       > a = `first; second`
-}

modules/translation/i18n/localestore.go

@@ -10,8 +10,8 @@ import (
 	"html/template"
 	"slices"
 
+	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
 )
 
 // This file implements the static LocaleStore that will not watch for changes
@@ -40,8 +40,8 @@ func NewLocaleStore() LocaleStore {
 	return &localeStore{localeMap: make(map[string]*locale), trKeyToIdxMap: make(map[string]int)}
 }
 
-// AddLocaleByIni adds locale by ini into the store
-func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, moreSource []byte) error {
+// AddLocaleByJSON adds locale by JSON into the store
+func (store *localeStore) AddLocaleByJSON(langName, langDesc string, source, moreSource []byte) error {
 	if _, ok := store.localeMap[langName]; ok {
 		return errors.New("lang has already been added")
 	}
@@ -52,28 +52,46 @@ func (store *localeStore) AddLocaleByIni(langName, langDesc string, source, more
 	l := &locale{store: store, langName: langName, idxToMsgMap: make(map[int]string)}
 	store.localeMap[l.langName] = l
 
-	iniFile, err := setting.NewConfigProviderForLocale(source, moreSource)
-	if err != nil {
-		return fmt.Errorf("unable to load ini: %w", err)
-	}
-
-	for _, section := range iniFile.Sections() {
-		for _, key := range section.Keys() {
-			var trKey string
-			if section.Name() == "" || section.Name() == "DEFAULT" {
-				trKey = key.Name()
-			} else {
-				trKey = section.Name() + "." + key.Name()
-			}
-			idx, ok := store.trKeyToIdxMap[trKey]
-			if !ok {
-				idx = len(store.trKeyToIdxMap)
-				store.trKeyToIdxMap[trKey] = idx
-			}
-			l.idxToMsgMap[idx] = key.Value()
+	addFunc := func(source []byte) error {
+		if len(source) == 0 {
+			return nil
 		}
+
+		values := make(map[string]any)
+		if err := json.Unmarshal(source, &values); err != nil {
+			return fmt.Errorf("unable to load json: %w", err)
+		}
+		for trKey, value := range values {
+			switch v := value.(type) {
+			case string:
+				idx, ok := store.trKeyToIdxMap[trKey]
+				if !ok {
+					idx = len(store.trKeyToIdxMap)
+					store.trKeyToIdxMap[trKey] = idx
+				}
+				l.idxToMsgMap[idx] = v
+			case map[string]any:
+				for key, val := range v {
+					idx, ok := store.trKeyToIdxMap[trKey+"."+key]
+					if !ok {
+						idx = len(store.trKeyToIdxMap)
+						store.trKeyToIdxMap[trKey+"."+key] = idx
+					}
+					l.idxToMsgMap[idx] = val.(string)
+				}
+			default:
+				return fmt.Errorf("unsupported value type %T for key %q", v, trKey)
+			}
+		}
+		return nil
 	}
 
+	if err := addFunc(source); err != nil {
+		return fmt.Errorf("unable to load json: %w", err)
+	}
+	if err := addFunc(moreSource); err != nil {
+		return fmt.Errorf("unable to load json: %w", err)
+	}
 	return nil
 }
 

modules/translation/translation.go

@@ -74,6 +74,9 @@ func InitLocales(ctx context.Context) {
 
 		localeData := make(map[string][]byte, len(localeNames))
 		for _, name := range localeNames {
+			if !strings.HasPrefix(name, "locale_") || !strings.HasSuffix(name, ".json") {
+				continue
+			}
 			localeData[name], err = options.Locale(name)
 			if err != nil {
 				log.Fatal("Failed to load %s locale file. %v", name, err)
@@ -90,14 +93,14 @@ func InitLocales(ctx context.Context) {
 			var localeDataBase []byte
 			if i == 0 && setting.Langs[0] != "en-US" {
 				// Only en-US has complete translations. When use other language as default, the en-US should still be used as fallback.
-				localeDataBase = localeData["locale_en-US.ini"]
+				localeDataBase = localeData["locale_en-US.json"]
 				if localeDataBase == nil {
-					log.Fatal("Failed to load locale_en-US.ini file.")
+					log.Fatal("Failed to load locale_en-US.json file.")
 				}
 			}
 
-			key := "locale_" + setting.Langs[i] + ".ini"
-			if err = i18n.DefaultLocales.AddLocaleByIni(setting.Langs[i], setting.Names[i], localeDataBase, localeData[key]); err != nil {
+			key := "locale_" + setting.Langs[i] + ".json"
+			if err = i18n.DefaultLocales.AddLocaleByJSON(setting.Langs[i], setting.Names[i], localeDataBase, localeData[key]); err != nil {
 				log.Error("Failed to set messages to %s: %v", setting.Langs[i], err)
 			}
 		}