fix: chinese special case and other tweaks

Author: nabalone

.github/workflows/release.yml

@@ -36,7 +36,7 @@ jobs:
 
       # find-language is a dependency of langauge-chooser-react-mui so it will have been built in the build step
       - name: Run tests
-        run: npx nx test @ethnolib/find-language
+        run: npx nx run-many --all --target=test
 
       - name: Set name and email for git so we can commit and tag
         run: |

components/language-chooser/common/find-language/searchForLanguage.ts

@@ -47,7 +47,7 @@ export function searchForLanguage(
   const allResultsFuse = new Fuse(languages as ILanguage[], {
     ...baseFuseOptions,
     ignoreLocation: true,
-    threshold: 0.2,
+    threshold: 0.3,
   });
   const allResults = allResultsFuse.search(queryString);
 

components/language-chooser/common/find-language/searchResultModifiers.spec.ts

@@ -1,13 +1,15 @@
-import { expect, it, describe, test, beforeEach } from "vitest";
+import { expect, it, describe, test, beforeEach, beforeAll } from "vitest";
 import {
   filterScripts,
   codeMatches,
   prioritizeLangByKeywords,
   filterLanguageCodes,
   substituteInSpecialEntry,
+  defaultSearchResultModifier,
 } from "./searchResultModifiers";
 import { ILanguage } from "./findLanguageInterfaces";
 import { createTestLanguageEntry } from "./testUtils";
+import { searchForLanguage } from "./searchForLanguage";
 
 describe("filter scripts", () => {
   it("should filter out scripts", () => {
@@ -175,4 +177,28 @@ describe("reordering entries to prioritize desired language when keywords are se
     );
     expect(reorderedResults[0].iso639_3_code).toEqual("tpi");
   });
+
+  describe("Chinese should be handled reasonably", () => {
+    let chineseResults: ILanguage[];
+    beforeAll(() => {
+      const chineseSearchString = "chinese";
+      chineseResults = defaultSearchResultModifier(
+        searchForLanguage(chineseSearchString),
+        chineseSearchString
+      );
+    });
+    it("top chinese result should have language subtag zh", () => {
+      expect(chineseResults[0].languageSubtag).toEqual("zh");
+    });
+    it("should only have one zh result", () => {
+      expect(
+        chineseResults.filter((r) => r.languageSubtag === "zh").length
+      ).toEqual(1);
+    });
+    it("zh result should have many alternative names listed", () => {
+      expect(
+        chineseResults.find((r) => r.languageSubtag === "zh")?.names.length
+      ).toBeGreaterThan(10);
+    });
+  });
 });

components/language-chooser/common/find-language/searchResultModifiers.ts

@@ -21,6 +21,16 @@ export function filterScripts(
   }));
 }
 
+export function modifyScripts(
+  scriptModifier: (value: IScript) => IScript,
+  results: ILanguage[]
+): ILanguage[] {
+  return results.map((result) => ({
+    ...result,
+    scripts: result.scripts.map(scriptModifier),
+  }));
+}
+
 const SCRIPT_CODES_TO_EXCLUDE = new Set([
   "Brai",
   "Zyyy",
@@ -72,6 +82,27 @@ function simplifyFrenchResult(results: ILanguage[]): ILanguage[] {
   return substituteInSpecialEntry("fra", getSpecialEntry, results);
 }
 
+function simplifyChineseResult(results: ILanguage[]): ILanguage[] {
+  function getSpecialEntry(result: ILanguage) {
+    return {
+      ...result,
+      regionNames: "", // clear the long and confusing list of region names
+      scripts: [
+        {
+          code: "Hans",
+          name: "Chinese (Simplified)",
+        } as IScript,
+        {
+          code: "Hant",
+          name: "Chinese (Traditional)",
+        } as IScript,
+        latinScriptData,
+      ],
+    } as ILanguage;
+  }
+  return substituteInSpecialEntry("zho", getSpecialEntry, results);
+}
+
 // Compare codes, ignoring any demarcation or casing
 // undefined does not match undefined
 export function codeMatches(
@@ -126,13 +157,26 @@ const ANCIENT_LANGUAGE_ENTRY_CODES = new Set([
   // Filter for deprecated, historical languages etc.
 ]);
 
+const SPECIAL_CASE_EXCLUDED_ENTRY_CODES = new Set([
+  "zhx", // I don't understand why this entry is in langtags.json. It is an ISO-639-5 (language collection) code covering the zho macrolanguage, has no Ethnologue entry, only listed script is Nshu
+  "cmn", // TODO when we implement macrolanguage handling, see if the situation is taken care of and we can remove this exception.
+  // In langtags.json, most chinese entries have iso639_3_code "zho" (which is the macrolanguage code) except zh-Brai-CN and zh-Hant-ES which have "cmn"
+  // so we end up with two search results and don't want to keep the "cmn" one
+]);
+
+const DEFAULT_EXCLUDED_ENTRY_CODES = new Set([
+  ...NOT_A_LANGUAGE_ENTRY_CODES,
+  ...ANCIENT_LANGUAGE_ENTRY_CODES,
+  ...SPECIAL_CASE_EXCLUDED_ENTRY_CODES,
+]);
+
 export function filterOutDefaultExcludedLanguages(
   results: ILanguage[]
 ): ILanguage[] {
   return filterLanguageCodes(
-    ((code) =>
-      !NOT_A_LANGUAGE_ENTRY_CODES.has(code) &&
-      !ANCIENT_LANGUAGE_ENTRY_CODES.has(code)) as (value: string) => boolean,
+    ((code) => !DEFAULT_EXCLUDED_ENTRY_CODES.has(code)) as (
+      value: string
+    ) => boolean,
     results
   );
 }
@@ -186,8 +230,15 @@ export function defaultSearchResultModifier(
     "fra",
     modifiedResults
   );
+  modifiedResults = prioritizeLangByKeywords(
+    ["chinese"],
+    searchString,
+    "zho", // TODO: if we implement improved macrolanguage handling, see if we should change this to cmn
+    modifiedResults
+  );
   modifiedResults = simplifyEnglishResult(modifiedResults);
   modifiedResults = simplifyFrenchResult(modifiedResults);
+  modifiedResults = simplifyChineseResult(modifiedResults);
   modifiedResults = filterOutDefaultExcludedLanguages(modifiedResults);
   modifiedResults = filterScripts(scriptFilter, modifiedResults);
   return modifiedResults;

components/language-chooser/react/common/language-chooser-react-hook/useLanguageChooser.ts

@@ -147,18 +147,56 @@ export const useLanguageChooser = (
         language.scripts.length === 1 ? language.scripts[0] : undefined
       );
       setCustomizableLanguageDetails({
-        displayName: stripDemarcation(
-          language.autonym || language.exonym || ""
-        ),
+        displayName: defaultDisplayName(language),
       } as ICustomizableLanguageDetails);
     }
   }
 
   function toggleSelectScript(script: IScript) {
+    // clicking on the selected script unselects it
     if (codeMatches(script.code, selectedScript?.code)) {
-      // clicking on the selected script unselects it
+      // if the display name is our automatically set Chinese (Simplified) for zh-Hans or Chinese (Traditional) for zh-Hant,
+      // then we want to reset to the default display name for Chinese with no selected script
+      if (
+        selectedLanguage?.languageSubtag === "zh" &&
+        ((codeMatches(script.code, "Hans") &&
+          customizableLanguageDetails.displayName === "Chinese (Simplified)") ||
+          (codeMatches(script.code, "Hant") &&
+            customizableLanguageDetails.displayName ===
+              "Chinese (Traditional)"))
+      ) {
+        setCustomizableLanguageDetails((d) => ({
+          ...d,
+          displayName: defaultDisplayName(selectedLanguage),
+        }));
+      }
       setSelectedScript(undefined);
     } else {
+      // selecting a script
+      if (
+        selectedLanguage?.languageSubtag === "zh" &&
+        customizableLanguageDetails.displayName ===
+          defaultDisplayName(selectedLanguage)
+      ) {
+        // automatically set the display name to Chinese (Simplified) or Chinese (Traditional) for zh-Hans or zh-Hant respectively
+        if (codeMatches(script.code, "Hans")) {
+          setCustomizableLanguageDetails(
+            (d) =>
+              ({
+                ...d,
+                displayName: "Chinese (Simplified)",
+              }) as ICustomizableLanguageDetails
+          );
+        } else if (codeMatches(script.code, "Hant")) {
+          setCustomizableLanguageDetails(
+            (d) =>
+              ({
+                ...d,
+                displayName: "Chinese (Traditional)",
+              }) as ICustomizableLanguageDetails
+          );
+        }
+      }
       setSelectedScript(script);
     }
   }
@@ -191,3 +229,7 @@ export const useLanguageChooser = (
     resetTo,
   } as ILanguageChooser;
 };
+
+export function defaultDisplayName(language: ILanguage) {
+  return stripDemarcation(language.autonym || language.exonym || "");
+}

components/language-chooser/react/language-chooser-react-mui/src/demos/DialogDemo.tsx

@@ -3,6 +3,7 @@ import { css } from "@emotion/react";
 import { defaultSearchResultModifier } from "@ethnolib/find-language";
 import { Button, Card, Dialog, Typography } from "@mui/material";
 import {
+  defaultDisplayName,
   IOrthography,
   parseLangtagFromLangChooser,
 } from "@ethnolib/language-chooser-react-hook";
@@ -22,10 +23,12 @@ export const DialogDemo: React.FunctionComponent<{
   // To demonstrate the ability to reopen to a desired state
   const initialSelection: IOrthography | undefined =
     parseLangtagFromLangChooser(props.initialLanguageTag || "");
-  if (props.initialCustomDisplayName !== undefined && initialSelection) {
+  if (initialSelection?.language) {
     initialSelection.customDetails = {
-      ...initialSelection.customDetails,
-      displayName: props.initialCustomDisplayName,
+      ...(initialSelection.customDetails || []),
+      displayName:
+        props.initialCustomDisplayName ??
+        defaultDisplayName(initialSelection.language),
     };
   }
 

components/language-chooser/react/language-chooser-react-mui/tsconfig.storybook.json

@@ -4,11 +4,6 @@
     "emitDecoratorMetadata": true,
     "outDir": ""
   },
-  "files": [
-    "../../../../node_modules/@nx/react/typings/styled-jsx.d.ts",
-    "../../../../node_modules/@nx/react/typings/cssmodule.d.ts",
-    "../../../../node_modules/@nx/react/typings/image.d.ts"
-  ],
   "exclude": [
     "src/**/*.spec.ts",
     "src/**/*.test.ts",