Collation rust updates - handle locales and other options (#482)

* ICU4X collation - mark rules as unsupported. Apply strength option.

* Cargo fmt

* Update collator.rs

* Fix selection on left mouse to unescape HTML entities

* Fix detail copy on left click to unescape HTML entitiees

* Add comment to fix locale handling

* fmt

* Adding locale in ICU4X collation

* Output the comparison numeric value

* cargo fmt

* Small refactor

* Gemini suggestions

* Fix boolean

* Handling more collation options

* Fix format

Author: sven-oly

executors/rust/src/collator.rs

@@ -8,29 +8,61 @@ use icu::collator::*;
 #[cfg(not(any(ver = "1.3", ver = "1.4", ver = "1.5", ver = "2.0-beta1")))]
 use icu::collator::options::*;
 
-use super::compat::{locale, pref};
+#[cfg(not(any(ver = "1.3", ver = "1.4", ver = "1.5", ver = "2.0-beta1")))]
+use icu::collator::preferences::{CollationCaseFirst as CaseFirst, CollationNumericOrdering};
+
+use super::compat::{pref, Locale};
 
 // Function runs comparison using collator
 pub fn run_collation_test(json_obj: &Value) -> Result<Value, String> {
-    // TODO: Handle errors of missing values and failures.
     let label = &json_obj["label"].as_str().unwrap();
-    let ignore_punctuation: &Option<bool> = &json_obj["ignorePunctuation"].as_bool();
+    let ignore_punctuation: Option<bool> = json_obj["ignorePunctuation"].as_bool();
     let str1: &str = json_obj["s1"].as_str().unwrap();
     let str2: &str = json_obj["s2"].as_str().unwrap();
 
-    // This may be missing
+    // These fields may be missing in tests
     let compare_option: Option<&str> = json_obj["compare_type"].as_str();
-
     let strength_option: Option<&str> = json_obj["strength"].as_str();
-
     let rules: Option<&str> = json_obj["rules"].as_str();
 
+    let alternate_option: Option<&str> = json_obj["alternate"].as_str();
+    let case_first_option: Option<&str> = json_obj["caseFirst"].as_str();
+    let case_level_option: Option<&str> = json_obj["caseLevel"].as_str();
+    let numeric_option: Option<&str> = json_obj["numeric"].as_str();
+    let reorder_option: Option<&str> = json_obj["reorder"].as_str();
+    let backwards_option: Option<&str> = json_obj["backwards"].as_str();
+    let max_variable_option: Option<&str> = json_obj["maxVariable"].as_str();
+
     #[cfg(any(ver = "1.3", ver = "1.4", ver = "1.5"))]
     let mut options = CollatorOptions::new();
     #[cfg(not(any(ver = "1.3", ver = "1.4", ver = "1.5")))]
     let mut options = CollatorOptions::default();
 
-    // TODO: Get and apply locale if given. Else use "und" or "en"
+    // Apply locale if given. Else use default locale.
+    // Replace "root" with default locale
+    let locale_name_opt = json_obj
+        .get("locale")
+        .map(|json_val| json_val.as_str().unwrap());
+    let langid = match locale_name_opt {
+        Some("root") | None => Locale::default(),
+        Some(other) => match other.parse() {
+            Ok(l) => l,
+            Err(_) => {
+                return Ok(json!({
+                    "label": label,
+                    "error_detail": other,
+                    "unsupported": "Unsupported locale",
+                    "error_type": "unsupported",
+                }))
+            }
+        },
+    };
+
+    #[cfg(any(ver = "1.3", ver = "1.4", ver = "1.5"))]
+    let preferences: &icu_provider::DataLocale = pref!(&langid);
+    #[cfg(not(any(ver = "1.3", ver = "1.4", ver = "1.5")))]
+    #[cfg_attr(ver = "2.0-beta1", allow(unused_mut))]
+    let mut preferences: CollatorPreferences = pref!(&langid);
 
     // Rules not yet supported.
     if rules.is_some() {
@@ -42,6 +74,16 @@ pub fn run_collation_test(json_obj: &Value) -> Result<Value, String> {
         }));
     }
 
+    if reorder_option.is_some() {
+        // Reordering is only supported by the -kr option of a locale
+        // See https://github.com/unicode-org/icu4x/issues/6033
+        return Ok(json!({
+            "label": label,
+            "unsupported": "reorder scripts",
+            "error_type": "unsupported",
+        }));
+    }
+
     // Use compare_type to get < or =.
     let compare_symbol = compare_option.and_then(|c| c.get(0..1)).unwrap_or("<");
 
@@ -63,18 +105,135 @@ pub fn run_collation_test(json_obj: &Value) -> Result<Value, String> {
         };
     };
 
+    if let Some(case_level) = case_level_option {
+        options.case_level = match case_level {
+            "off" => Some(CaseLevel::Off),
+            "on" => Some(CaseLevel::On),
+            _ => {
+                return Ok(json!({
+                    "label": label,
+                    "error_detail": {"caseLevel": case_level},
+                    "unsupported": "caseLevel",
+                    "error_type": "unsupported",
+                }));
+            }
+        }
+    };
+
+    if let Some(case_first) = case_first_option {
+        #[cfg(not(any(ver = "1.3", ver = "1.4", ver = "1.5", ver = "2.0-beta1")))]
+        {
+            preferences.case_first = match case_first {
+                "off" => Some(CaseFirst::False),
+                "lower" => Some(CaseFirst::Lower),
+                "upper" => Some(CaseFirst::Upper),
+                _ => {
+                    return Ok(json!({
+                        "label": label,
+                        "error_detail": {"caseFirst": case_first},
+                        "unsupported": "caseFirst",
+                        "error_type": "unsupported",
+                    }));
+                }
+            }
+        }
+        #[cfg(any(ver = "1.3", ver = "1.4", ver = "1.5", ver = "2.0-beta1"))]
+        {
+            options.case_first = match case_first {
+                "off" => Some(CaseFirst::Off),
+                "lower" => Some(CaseFirst::LowerFirst),
+                "upper" => Some(CaseFirst::UpperFirst),
+                _ => {
+                    return Ok(json!({
+                        "label": label,
+                        "error_detail": {"caseFirst": case_first},
+                        "unsupported": "caseFirst",
+                        "error_type": "unsupported",
+                    }));
+                }
+            }
+        }
+    };
+
+    if let Some(backwards) = backwards_option {
+        options.backward_second_level = match backwards {
+            "off" => Some(BackwardSecondLevel::Off),
+            "on" => Some(BackwardSecondLevel::On),
+            _ => {
+                return Ok(json!({
+                    "label": label,
+                    "error_detail": {"backwards": backwards},
+                    "unsupported": "backwards",
+                    "error_type": "unsupported",
+                }));
+            }
+        }
+    };
+
+    // Numeric sort order.
+    // CollatorPreferences in beta2 vs. Enum Numeric
+    if let Some(numeric) = numeric_option {
+        #[cfg(not(any(ver = "1.3", ver = "1.4", ver = "1.5", ver = "2.0-beta1")))]
+        {
+            preferences.numeric_ordering = match numeric {
+                "off" => Some(CollationNumericOrdering::False),
+                "on" => Some(CollationNumericOrdering::True),
+                _ => {
+                    return Ok(json!({
+                        "label": label,
+                        "error_detail": {"numeric": numeric},
+                        "unsupported": "numeric",
+                        "error_type": "unsupported",
+                    }));
+                }
+            }
+        };
+        // !!! TODO: handle before 2.0beta2
+    };
+
+    if let Some(alternate) = alternate_option {
+        options.alternate_handling = match alternate {
+            "shifted" => Some(AlternateHandling::Shifted),
+            "non-ignorable" => Some(AlternateHandling::NonIgnorable),
+            _ => {
+                return Ok(json!({
+                    "label": label,
+                    "error_detail": {"alternate": alternate},
+                    "unsupported": "alternate",
+                    "error_type": "unsupported",
+                }));
+            }
+        }
+    };
+
+    if let Some(max_variable) = max_variable_option {
+        options.max_variable = match max_variable {
+            "space" => Some(MaxVariable::Space),
+            "punctuation" => Some(MaxVariable::Punctuation),
+            "symbol" => Some(MaxVariable::Symbol),
+            "currency" => Some(MaxVariable::Currency),
+            _ => {
+                return Ok(json!({
+                    "label": label,
+                    "error_detail": {"maxVariable": max_variable},
+                    "unsupported": "maxVariable",
+                    "error_type": "unsupported",
+                }));
+            }
+        }
+    };
+
     // Ignore punctuation only if using shifted test.
     if let Some(ip) = ignore_punctuation {
-        if *ip {
+        if ip {
             options.alternate_handling = Some(AlternateHandling::Shifted);
         }
     }
 
     // TODO !! Iterate to find actual level of comparison, then look
     // at compare type (1, 2, 3, 4, i, c) to see if it matches
 
-    let collator = Collator::try_new(pref!(locale!("en")), options).unwrap();
-
+    let collator = Collator::try_new(preferences, options).map_err(|e| e.to_string())?;
     let comparison = collator.compare(str1, str2);
 
     let result_string = match compare_symbol {
@@ -92,19 +251,21 @@ pub fn run_collation_test(json_obj: &Value) -> Result<Value, String> {
         }
     };
 
-    let mut comparison_number: i16 = 0;
-    if comparison == Ordering::Less {
-        comparison_number = -1;
-    } else if comparison == Ordering::Greater {
-        comparison_number = 1;
-    }
+    let comparison_number: i16 = match comparison {
+        Ordering::Less => -1,
+        Ordering::Greater => 1,
+        Ordering::Equal => 0,
+    };
 
-    // TODO: Convert comparison to "<", "=", or ">"
     let json_result = json!({
         "label": label,
         "result": result_string,
-        "compare_result": comparison_number,
-        "actual_options": format!("{options:?}")
+        "actual_options": {
+            "options": format!("{options:?}, {preferences:?}"),
+            "compared_result": comparison_number,
+            "s1": str1,
+            "s2": str2,
+        }
     });
     Ok(json_result)
 }