Merge branch 'main' into update-version-v0.9.7

Author: ManyTheFish

charabia/benches/bench.rs

@@ -17,6 +17,7 @@ static DATA_SET: &[((usize, Script, Language), &str)] = &[
     ((130, Script::Greek, Language::Ell), "Οι θερμοκρασίες είναι σπάνια υπερβολικές στις παραθαλάσσιες περιοχές."),
     ((132, Script::Khmer, Language::Khm), "ធ្វេីមនុស្សត្រូវចេះស្រលាញ់នឹងជួយគ្នាទៅវិញទៅមក ព្រោះពិភពលោកនេះមានទុកច្រេីនហេីយគួយតែមានអំពេីល្អច្រេីនមិនថាជួយបាន១រឺ២នាក់ច្រេីនរឺតិចទេ៕"),
     ((132, Script::Arabic, Language::Ara), "اللُّغَةُ العربية هي أكثر اللغات السامية تحدثا، ومن أكثر اللغات انتشارا"),
+    ((128, Script::Arabic, Language::Pes), "قنات قصبه شهر گناباد عمیق‌ترین و قدیمی‌ترین کاریز جهان است."),
     ((134, Script::Arabic, Language::Vie), "Các nhà nước trong lịch sử Việt Nam có những quốc hiệu khác nhau như Xích Quỷ, Văn Lang, Đại Việt, Đại"),
     ((131, Script::Latin, Language::Deu), "Deutschland vereint Alpen, Küsten und Städte wie Berlin. Kultur und Geschichte prägen das Land, das Natur und Moderne verbindet."),
 
@@ -31,6 +32,7 @@ static DATA_SET: &[((usize, Script, Language), &str)] = &[
     ((364, Script::Greek, Language::Ell), "Η άνοιξη έχει μικρή διάρκεια, διότι ο μεν χειμώνας είναι όψιμος, το δε καλοκαίρι αρχίζει πρώιμα. Το φθινόπωρο είναι μακρύ και θερμό και πολλές φορές παρατείνεται στη νότια Ελλάδα και τα νησιά μέχρι τα"),
     ((327, Script::Khmer, Language::Khm), "រឿងពីរដែលមនុស្សហាមចិត្តខ្លួនឯងមិនបានគឺ សើច និង ស្រឡាញ់។ តែសម្រាប់ខ្ញុំ ប្រាក់ ចន្ទធីតា រឿងមួយទៀតដែលខ្ញុំហាមចិត្តខ្លួនឯងមិនបាននោះ គឺញ៉ាំ គេគ្រប់គ្នាពេលខូចចិត្តបាយទឹកមិនបានទេ តែខ្ញុំពេលខូចចិត្តដឹងតែឃ្លាន ញ៉ាំច្រើនឬតិចក៏អាស្រ័យលើថាទំហំនៃការខូចចិត្តខ្លាំងឬខ្សោយ។"),
     ((366, Script::Arabic, Language::Ara), "العربية لغةٌ رسمية في كل دول الوطن العربي (إضافة إلى كونها لغة رسمية في تشاد وإريتريا). وهي إحدى اللغات الرسمية الست في منظمة الأمم المتحدة، ويُحتفل بالعربية في 18 ديسمبر كذكرى اعتمادها في الأمم المتحدة."),
+    ((366, Script::Arabic, Language::Pes), "فارسی یکی از زبان‌های هندواروپایی و زبان رسمی ایران، افغانستان (با نام دری) و تاجیکستان (با نام تاجیکی) است. این زبان دارای پیشینه‌ای کهن و ادبیاتی غنی می‌باشد و در طول تاریخ، شاعران و نویسندگان بزرگی به این زبان آثار خود را خلق کرده‌اند. فارسی همچنین یکی از زبان‌های مهم منطقه به شمار می‌رود و در سازمان‌های بین‌المللی نیز جایگاه ویژه‌ای دارد."),
     ((365, Script::Latin, Language::Vie), "Lãnh thổ Việt Nam xuất hiện con người sinh sống từ thời đại đồ đá cũ, khởi đầu với các nhà nước Văn Lang, Âu Lạc. Âu Lạc bị nhà Triệu ở phương Bắc thôn tính vào đầu thế kỷ thứ 2 TCN sau đó là thời kỳ Bắc thuộc kéo dài hơn một thiên niên kỷ.Chế độ quân chủ độc lập"),
     ((354, Script::Latin, Language::Deu), "Magdeburg, die Hauptstadt Sachsen-Anhalts, beeindruckt mit dem Magdeburger Dom, dem Jahrtausendturm im Elbauenpark und dem Wasserstraßenkreuz. Der Domplatz ist umgeben von Bauwerken, wie dem Hundertwasserhaus. Der Elbauenpark bietet viele Freizeitmöglichkeiten, während die Magdeburger Börde für fruchtbare Ackerflächen für z.B. Zuckerrüben bekannt ist."),
 ];

charabia/src/detection/chars.rs

@@ -42,6 +42,24 @@ pub(crate) fn is_arabic(ch: char) -> bool {
     )
 }
 
+// Based on: https://en.wikipedia.org/wiki/Persian_alphabet
+pub(crate) fn is_persian(ch: char) -> bool {
+    matches!(
+        ch,
+        // Persian-specific letters
+        | '\u{067E}' // Peh
+        | '\u{0686}' // Tcheh
+        | '\u{0698}' // Jeh
+        | '\u{06A9}' // Keheh (Persian Kaf)
+        | '\u{06AF}' // Gaf
+        | '\u{06CC}' // Farsi Yeh
+        | '\u{06C0}' // Yeh with Hamza above (used in Dari)
+
+        // Persian digits
+        | '\u{06F0}'..='\u{06F9}' // Zero to nine
+    )
+}
+
 // Based on https://en.wikipedia.org/wiki/Devanagari#Unicode
 pub(crate) fn is_devanagari(ch: char) -> bool {
     matches!(ch, '\u{0900}'..='\u{097F}' | '\u{A8E0}'..='\u{A8FF}' | '\u{1CD0}'..='\u{1CFF}')
@@ -287,4 +305,17 @@ mod tests {
         assert!(is_hebrew('ׇ'));
         assert!(!is_hebrew('s'));
     }
+
+    #[test]
+    fn test_is_persian() {
+        assert!(is_persian('پ')); // Peh
+        assert!(is_persian('ژ')); // Zheh
+        assert!(is_persian('گ')); // Gaf
+        assert!(is_persian('ک')); // Kaf
+        assert!(is_persian('ی')); // Yeh
+        assert!(is_persian('۱')); // Latin '1'
+        assert!(is_persian('۲')); // Latin '2'
+
+        assert!(!is_persian('z')); // Latin 'z'
+    }
 }

charabia/src/detection/script_language.rs

@@ -188,7 +188,7 @@ impl From<char> for Script {
             Script::Latin
         } else if chars::is_cyrillic(other) {
             Script::Cyrillic
-        } else if chars::is_arabic(other) {
+        } else if chars::is_arabic(other) || chars::is_persian(other) {
             Script::Arabic
         } else if chars::is_devanagari(other) {
             Script::Devanagari

charabia/src/normalizer/arabic.rs

@@ -27,15 +27,15 @@ impl CharNormalizer for ArabicNormalizer {
 fn normalize_arabic_char(c: char) -> Option<CharOrStr> {
     match c {
         'ـ' => None,
-        'ٱ' => Some('ا'.into()),
+        'أ' | 'إ' | 'آ' | 'ٱ' => Some('ا'.into()), // All Alef variants to Alef
         'ى' => Some('ي'.into()),
         'ة' => Some('ه'.into()),
         _ => Some(c.into()),
     }
 }
 
 fn is_shoud_normalize(c: char) -> bool {
-    matches!(c, 'ـ' | 'ٱ' | 'ى' | 'ة')
+    matches!(c, 'ـ' | 'أ' | 'إ' | 'آ' | 'ٱ' | 'ى' | 'ة')
 }
 
 #[cfg(test)]

charabia/src/normalizer/mod.rs

@@ -1,7 +1,5 @@
 use std::borrow::Cow;
 
-use once_cell::sync::Lazy;
-
 pub use self::ae_oe_normalizer::AeOeNormalizer;
 pub use self::arabic::ArabicNormalizer;
 #[cfg(feature = "chinese-normalization")]
@@ -15,6 +13,7 @@ use self::greek::GreekNormalizer;
 pub use self::japanese::JapaneseNormalizer;
 pub use self::lowercase::LowercaseNormalizer;
 use self::nonspacing_mark::NonspacingMarkNormalizer;
+pub use self::persian::PersianNormalizer;
 use self::quote::QuoteNormalizer;
 #[cfg(feature = "swedish-recomposition")]
 use self::swedish_recomposition::SwedishRecompositionNormalizer;
@@ -24,6 +23,7 @@ pub use self::turkish::TurkishNormalizer;
 pub use self::vietnamese::VietnameseNormalizer;
 use crate::segmenter::SegmentedTokenIter;
 use crate::Token;
+use once_cell::sync::Lazy;
 
 mod arabic;
 #[cfg(feature = "chinese-normalization")]
@@ -46,6 +46,7 @@ mod turkish;
 mod vietnamese;
 
 mod ae_oe_normalizer;
+mod persian;
 
 /// List of [`Normalizer`]s used by [`Normalize::normalize`] that are not considered lossy.
 pub static NORMALIZERS: Lazy<Vec<Box<dyn Normalizer>>> = Lazy::new(|| {
@@ -55,6 +56,7 @@ pub static NORMALIZERS: Lazy<Vec<Box<dyn Normalizer>>> = Lazy::new(|| {
         Box::new(SwedishRecompositionNormalizer),
         Box::new(ControlCharNormalizer),
         Box::new(Classifier),
+        Box::new(PersianNormalizer),
     ]
 });