Proper arabic analyzer

sunnah-com · Feb 22, 2024 · 378a833 · 378a833
1 parent 86673e9
commit 378a833
Showing 1 changed file with 16 additions and 6 deletions.
diff --git a/main.py b/main.py
@@ -45,15 +45,17 @@ def create_and_update_index(index_name, documents, fields_to_not_index):
                             "stemmer",
                         ],
                     },
-                    "arabic": {
-                        "tokenizer": "standard",
+                    "rebuilt_arabic": {
+                        "tokenizer":  "standard",
                         "char_filter": ["html_strip"],
                         "filter": [
                             "lowercase",
-                            "stop",
-                            "stemmer",
-                        ],
-                    },
+                            "decimal_digit",
+                            "arabic_stop",
+                            "arabic_normalization",
+                            "arabic_stemmer"
+                        ]
+                    }
                 },
                 "filter": {
                     # 2-3 word shingles for better suggestions
@@ -67,6 +69,14 @@ def create_and_update_index(index_name, documents, fields_to_not_index):
                         "lenient": True,
                         "synonyms_path": "synonyms.txt",
                     },
+                    "arabic_stemmer": {
+                        "type":       "stemmer",
+                        "language":   "arabic"
+                    },
+                    "arabic_stop": {
+                        "type":       "stop",
+                        "stopwords":  "_arabic_" 
+                    },
                 },
             },
         }