Avoid toLowerCasing İ (\u0130) - it changes

(closes osmlab#8261)

This was tricky because you'd think that case insensitive regex /i would catch
both upper and lower case variants of this, but it doesn't.

Then, I tried to match both variants with a regex like '^(İ|i̇)nşaat malları$',
but toLowerCasing *that* regex in our file_tree writing code was changing the 'İ'.

So for now, our build scripts can just avoid toLowerCasing a string with a 'İ' in it.

Author: bhousel

data/brands/shop/doityourself.json

@@ -3,9 +3,9 @@
     "path": "brands/shop/doityourself",
     "exclude": {
       "generic": [
+        "^(İ|i̇)nşaat malları$",
         "^(сантехника|стройматериалы)$",
         "^doityourself$",
-        "^i̇nşaat malları$",
         "^будматеріали$",
         "^хоз(товары)$"
       ],
@@ -736,16 +736,6 @@
         "shop": "doityourself"
       }
     },
-    {
-      "displayName": "İnşaat Malları",
-      "id": "insaatmallari-092aaf",
-      "locationSet": {"include": ["001"]},
-      "tags": {
-        "brand": "İnşaat Malları",
-        "name": "İnşaat Malları",
-        "shop": "doityourself"
-      }
-    },
     {
       "displayName": "jem & fix",
       "id": "jemandfix-27539e",

lib/file_tree.js

@@ -305,13 +305,18 @@ write: (cache) => {
   });
 
 
-  function _clean(val) {
-    if (typeof val !== 'string') return val;
-    return val.trim();
+  function _clean(s) {
+    if (typeof s !== 'string') return s;
+    return s.trim();
   }
-  function _cleanLower(val) {
-    if (typeof val !== 'string') return val;
-    return val.trim().toLowerCase();
+
+  function _cleanLower(s) {
+    if (typeof s !== 'string') return s;
+    if (/İ/.test(s)) {  // Avoid toLowerCasing this one, it changes - #8261
+      return s.trim();
+    } else {
+      return s.trim().toLowerCase();
+    }
   }
 },
 

lib/simplify.js

@@ -9,7 +9,7 @@ export function simplify(str) {
   return diacritics.remove(
     str
       .replace(/&/g, 'and')
-      .replace(/İ/ig, 'i')   // for BİM, İşbank - #5017
+      .replace(/(İ|i̇)/ig, 'i')   // for BİM, İşbank - #5017, #8261
       .replace(/[\s\-=_!"#%'*{},.\/:;?\(\)\[\]@\\$\^*+<>«»~`’\u00a1\u00a7\u00b6\u00b7\u00bf\u037e\u0387\u055a-\u055f\u0589\u05c0\u05c3\u05c6\u05f3\u05f4\u0609\u060a\u060c\u060d\u061b\u061e\u061f\u066a-\u066d\u06d4\u0700-\u070d\u07f7-\u07f9\u0830-\u083e\u085e\u0964\u0965\u0970\u0af0\u0df4\u0e4f\u0e5a\u0e5b\u0f04-\u0f12\u0f14\u0f85\u0fd0-\u0fd4\u0fd9\u0fda\u104a-\u104f\u10fb\u1360-\u1368\u166d\u166e\u16eb-\u16ed\u1735\u1736\u17d4-\u17d6\u17d8-\u17da\u1800-\u1805\u1807-\u180a\u1944\u1945\u1a1e\u1a1f\u1aa0-\u1aa6\u1aa8-\u1aad\u1b5a-\u1b60\u1bfc-\u1bff\u1c3b-\u1c3f\u1c7e\u1c7f\u1cc0-\u1cc7\u1cd3\u2000-\u206f\u2cf9-\u2cfc\u2cfe\u2cff\u2d70\u2e00-\u2e7f\u3001-\u3003\u303d\u30fb\ua4fe\ua4ff\ua60d-\ua60f\ua673\ua67e\ua6f2-\ua6f7\ua874-\ua877\ua8ce\ua8cf\ua8f8-\ua8fa\ua92e\ua92f\ua95f\ua9c1-\ua9cd\ua9de\ua9df\uaa5c-\uaa5f\uaade\uaadf\uaaf0\uaaf1\uabeb\ufe10-\ufe16\ufe19\ufe30\ufe45\ufe46\ufe49-\ufe4c\ufe50-\ufe52\ufe54-\ufe57\ufe5f-\ufe61\ufe68\ufe6a\ufe6b\ufeff\uff01-\uff03\uff05-\uff07\uff0a\uff0c\uff0e\uff0f\uff1a\uff1b\uff1f\uff20\uff3c\uff61\uff64\uff65]+/g,'')
       .toLowerCase()
   );

scripts/build_index.js

@@ -116,7 +116,14 @@ function loadConfig() {
       data.replacements = sortObject(data.replacements);
 
     } else if (which === 'genericWords') {
-      data.genericWords = data.genericWords.map(s => s.toLowerCase()).sort(withLocale);
+      data.genericWords = data.genericWords.map(s => {
+        if (/İ/.test(s)) {   // Avoid toLowerCasing this one, it changes - #8261
+          return s.trim();
+        } else {
+          return s.trim().toLowerCase();
+        }
+      })
+      .sort(withLocale);
     }
 
     // Lowercase and sort the files for consistency, save them that way.

tests/simplify.test.js

@@ -46,6 +46,12 @@ test('simplify', t => {
     t.end();
   });
 
+  t.test('replaces İ (0130) or i̇ (0069 0307) with i', t => {   // #5017, #8261 for examples
+    t.equal(simplify('İnşaat'), 'insaat');
+    t.equal(simplify('i̇nşaat'), 'insaat');
+    t.end();
+  });
+
   t.test('returns empty string if no input', t => {
     t.equal(simplify(), '');
     t.equal(simplify(null), '');