snowball-tamil.patch

Index: algorithms/tamil/stem_Unicode.sbl
===================================================================
--- algorithms/tamil/stem_Unicode.sbl	(revision 0)
+++ algorithms/tamil/stem_Unicode.sbl	(working copy)
@@ -0,0 +1,400 @@
+/*
+* Affix stripping stemming algorithm for Tamil
+* By Damodharan Rajalingam
+*/
+
+stringescapes {}
+
+/* Aytham */
+stringdef aytham hex '0B83'
+
+/* Uyir - independent vowels */
+stringdef a hex '0B85'
+stringdef aa hex '0B86'
+stringdef i hex '0B87'
+stringdef ii hex '0B88'
+stringdef u hex '0B89'
+stringdef uu hex '0B8A'
+stringdef e hex '0B8E'
+stringdef ee hex '0B8F'
+stringdef ai hex '0B90'
+stringdef o hex '0B92'
+stringdef oo hex '0B93'
+stringdef au hex '0B94'
+
+/* Consonants */
+stringdef ka hex '0B95'
+stringdef nga hex '0B99'
+stringdef ca hex '0B9A'
+stringdef ja hex '0B9C'
+stringdef nya hex '0B9E'
+stringdef tta hex '0B9F'
+stringdef nna hex '0BA3'
+stringdef ta hex '0BA4'
+stringdef tha hex '0BA4'
+stringdef na hex '0BA8'
+stringdef nnna hex '0BA9'
+stringdef pa hex '0BAA'
+stringdef ma hex '0BAE'
+stringdef ya hex '0BAF'
+stringdef ra hex '0BB0'
+stringdef rra hex '0BB1'
+stringdef la hex '0BB2'
+stringdef lla hex '0BB3'
+stringdef llla hex '0BB4'
+stringdef zha hex '0BB4'
+stringdef va hex '0BB5'
+
+/* Vatamozi - borrowed */
+stringdef sha hex '0BB6'
+stringdef ssa hex '0BB7'
+stringdef sa hex '0BB8'
+stringdef ha hex '0BB9'
+
+
+/* Dependent vowel signs (kombu etc.) */
+stringdef vs_aa hex '0BBE'
+stringdef vs_i hex '0BBF'
+stringdef vs_ii hex '0BC0'
+stringdef vs_u hex '0BC1'
+stringdef vs_uu hex '0BC2'
+stringdef vs_e hex '0BC6'
+stringdef vs_ee hex '0BC7'
+stringdef vs_ai hex '0BC8'
+stringdef vs_o hex '0BCA'
+stringdef vs_oo hex '0BCB'
+stringdef vs_au hex '0BCC'
+
+/* Pulli */
+stringdef pulli hex '0BCD'
+
+/* AU length markk */
+stringdef au_lmark hex '0BD7'
+
+
+routines (
+ remove_plural_suffix
+ remove_question_suffixes
+ remove_question_prefixes
+ remove_pronoun_prefixes
+ remove_command_suffixes
+ remove_um
+ remove_vetrumai_urupukal
+ fix_va_start
+ fix_ending
+ fix_endings
+ remove_tense_suffix
+ remove_tense_suffixes
+ remove_common_word_endings
+ strlen
+ has_min_length
+)
+
+integers (mark mark2)
+
+externals ( stem )
+
+groupings (
+ suttezhuthu
+ q_suffixes
+ q_prefixes
+ word_starter
+ vallinam
+ mellinam
+ itaiyinam
+ uyir
+ vowel_signs
+)
+
+booleans (
+ found_vallinam_doubling
+ found_a_match
+ found_vetrumai_urupu
+ found_wrong_ending
+ was_stripped
+)
+
+integers (
+ length
+)
+
+define q_suffixes '{vs_aa}{vs_oo}{vs_ee}'
+define q_prefixes '{e}' /* not clear about {ya}{vs_aa} and {ee} */
+define word_starter '{ka}{ca}{tha}{va}{na}{pa}{ma}{ya}{nga}{nya}'
+define suttezhuthu '{a}{i}{u}'
+define vallinam '{ka}{ca}{tta}{tha}{pa}{rra}'
+define mellinam '{nga}{nya}{nna}{na}{ma}{nnna}'
+define itaiyinam '{ya}{ra}{la}{va}{zha}{lla}'
+define vowel_signs '{vs_aa}{vs_i}{vs_ii}{vs_e}{vs_ee}{vs_u}{vs_uu}{vs_ai}'
+define uyir '{a}{aa}{i}{ii}{u}{uu}{e}{ee}{ai}{o}{oo}{au}'
+define strlen as (
+ // size in snowball returns bytes which is not very useful for utf8
+ // this routine gives the number of characters
+ $length = 0
+ do repeat (next ($length = length+1))
+)
+define has_min_length as (
+ strlen
+ $length > 4
+)
+define fix_va_start as (
+ (try '{va}{vs_oo}' and [ '{va}{vs_oo}' ] <- '{oo}' ) or
+ (try '{va}{vs_o}' and [ '{va}{vs_o}' ] <- '{o}' ) or
+ (try '{va}{vs_u}' and [ '{va}{vs_u}' ] <- '{u}' ) or
+ (try '{va}{vs_uu}' and [ '{va}{vs_uu}' ] <- '{uu}' )
+)
+define fix_endings as (
+ set found_wrong_ending
+ repeat (found_wrong_ending (do fix_ending))
+)
+define remove_question_prefixes as (
+ [ ('{e}' ) ('{ka}' or '{ca}' or '{tha}' or '{va}' or '{na}' or '{pa}' or '{ma}' or '{ya}' or '{nga}' or '{nya}') '{pulli}' ] delete
+ do fix_va_start
+)
+define fix_ending as (
+ unset found_wrong_ending
+ strlen
+ $length > 3
+ backwards (
+  ( ['{na}{pulli}' or '{na}{pulli}{ta}' or '{na}{pulli}{ta}{pulli}' ] delete )
+  or
+  ( ['{ya}{pulli}' test ('{vs_ai}' or '{vs_i}' or '{vs_ii}') ] delete )
+  or
+  ( [ '{tta}{pulli}{pa}{pulli}' or '{tta}{pulli}{ka}{pulli}' ] <- '{lla}{pulli}' )
+  or
+  ( [ '{nnna}{pulli}{rra}{pulli}' ] <- '{la}{pulli}' )
+  or
+//		( [ '{rra}{pulli}{ka}{pulli}'  or '{nnna}{pulli}{nnna}{pulli}' ] <- '{la}{pulli}'  )
+  ( [ '{rra}{pulli}{ka}{pulli}' ] <- '{la}{pulli}' )
+  or
+  ( [ '{tta}{pulli}{tta}{pulli}' ] <- '{tta}{vs_u}' )
+  or
+  ( found_vetrumai_urupu [ '{ta}{pulli}{ta}{pulli}' (test not '{vs_ai}') ] <- '{ma}{pulli}' ] )
+  or
+  ( [ '{vs_u}{ka}{pulli}' or '{vs_u}{ka}{pulli}{ka}{pulli}' ] <- '{pulli}' )
+  or
+  ( [ '{pulli}' ('{ka}' or '{ca}' or '{tta}' or '{tha}' or '{pa}' or '{rra}') '{pulli}' ('{ka}' or '{ca}' or '{tta}' or '{tha}' or '{pa}' or '{rra}') ] delete )
+  or
+  ( [ '{vs_u}{ka}{pulli}' ] <- '{pulli}' )
+  or
+  ( [ '{pulli}' ('{ka}' or '{ca}' or '{tta}' or '{tha}' or '{pa}' or '{rra}') ] delete )
+  or
+  ( [ '{pulli}' (('{ya}' or '{ra}' or '{la}' or '{va}' or '{zha}' or '{lla}') or ('{nga}' or '{nya}' or '{nna}' or '{na}' or '{ma}' or '{nnna}')) '{pulli}' ] <- '{pulli}' )
+  or
+  ( [ '{va}' or '{ya}' or '{va}{pulli}' ] delete )
+  or
+  ( [ '{nnna}{vs_u}' (test not ('{vs_aa}' or '{vs_i}' or '{vs_ii}' or '{vs_e}' or '{vs_ee}' or '{vs_u}' or '{vs_uu}' or '{vs_ai}')) ] delete )
+  or
+  ( [ '{nga}{pulli}' (test not '{vs_ai}')] <- '{ma}{pulli}' )
+  or
+  ( [ '{nga}{pulli}' ] delete )
+  or
+  ( [ '{pulli}' (test (('{vs_aa}' or '{vs_i}' or '{vs_ii}' or '{vs_e}' or '{vs_ee}' or '{vs_u}' or '{vs_uu}' or '{vs_ai}') or '{pulli}')) ] delete )
+  )
+
+ set found_wrong_ending // If any of above test pass set the flag
+)
+define remove_pronoun_prefixes as (
+ unset found_a_match
+ [ ('{a}' or '{i}' or '{u}') ('{ka}' or '{ca}' or '{tha}' or '{va}' or '{na}' or '{pa}' or '{ma}' or '{ya}' or '{nga}' or '{nya}') '{pulli}' ] delete
+ (set found_a_match)
+ do fix_va_start
+)
+define remove_plural_suffix as (
+ unset found_a_match
+ backwards (
+  ( [ '{vs_u}{nga}{pulli}{ka}{lla}{pulli}' (test not ('{ka}' or '{ca}' or '{tta}' or '{tha}' or '{pa}' or '{rra}')) ] <- '{pulli}' ) or
+  ( [ '{rra}{pulli}{ka}{lla}{pulli}' ] <- '{la}{pulli}' ) or
+  ( [ '{tta}{pulli}{ka}{lla}{pulli}' ] <- '{lla}{pulli}' ) or
+  ( [ '{ka}{lla}{pulli}' ] delete )
+  (set found_a_match)
+ 
+ )
+)
+define remove_question_suffixes as (
+ has_min_length
+ unset found_a_match
+ backwards (
+  do (
+   [ ( '{vs_oo}' or '{vs_ee}' or '{vs_aa}') ] <- '{pulli}'
+   (set found_a_match)
+  
+  )
+ )
+ do fix_endings
+)
+define remove_command_suffixes as (
+ has_min_length
+ unset found_a_match
+ backwards (
+  [ ('{pa}{vs_i}' or '{va}{vs_i}') ] delete
+  (set found_a_match)
+ 
+ )
+)
+define remove_um as (
+ unset found_a_match
+ has_min_length
+ backwards ( [ '{vs_u}{ma}{pulli}' ] <- '{pulli}'
+    (set found_a_match)
+    )
+ do fix_ending
+)
+define remove_common_word_endings as (
+ // These are not suffixes actually but are 
+ // some words that are attached to other words
+ // but can be removed for stemming
+
+ unset found_a_match
+ has_min_length
+ backwards (
+  test ( [ '{vs_u}{tta}{nnna}{pulli}' or
+     '{vs_i}{la}{pulli}{la}{vs_ai}' or
+       '{vs_i}{tta}{ma}{pulli}' or '{vs_i}{nnna}{pulli}{rra}{vs_i}' or
+     '{vs_aa}{ka}{vs_i}' or '{vs_aa}{ka}{vs_i}{ya}' or
+     '{vs_e}{nnna}{pulli}{rra}{vs_u}' or '{vs_u}{lla}{pulli}{lla}' or
+     '{vs_u}{tta}{vs_ai}{ya}' or '{vs_u}{tta}{vs_ai}' or
+     '{vs_e}{nnna}{vs_u}{ma}{pulli}' or
+     ('{la}{pulli}{la}' test (not ('{vs_aa}' or '{vs_i}' or '{vs_ii}' or '{vs_e}' or '{vs_ee}' or '{vs_u}' or '{vs_uu}' or '{vs_ai}'))) or
+     '{vs_e}{nnna}' or '{vs_aa}{ka}{vs_i}' ] <- '{pulli}'
+     (set found_a_match)
+     )
+  or
+  test ( [ '{pa}{tta}{vs_u}' or
+     '{pa}{tta}{pulli}{tta}' or '{pa}{tta}{pulli}{tta}{vs_u}' or
+     '{pa}{tta}{pulli}{tta}{ta}{vs_u}' or
+     '{pa}{tta}{pulli}{tta}{nna}' or '{ka}{vs_u}{ra}{vs_i}{ya}' or
+     '{ka}{vs_u}{ra}{vs_i}{ya}' or '{pa}{rra}{pulli}{rra}{vs_i}' or
+     '{va}{vs_i}{tta}{vs_u}' or '{va}{vs_i}{tta}{pulli}{tta}{vs_u}' or
+     '{pa}{tta}{vs_i}{ta}{vs_aa}{nnna}' or
+     '{pa}{tta}{vs_i}' or '{ta}{vs_aa}{nnna}' or
+     '{vs_e}{la}{pulli}{la}{vs_aa}{ma}{pulli}'
+    ] delete
+    (set found_a_match)
+    )
+ )
+ do fix_endings
+)
+define remove_vetrumai_urupukal as (
+ unset found_a_match
+ unset found_vetrumai_urupu
+ has_min_length
+ backwards (
+  (
+    test ( ['{nnna}{vs_ai}'] delete )
+   or
+    test ([ ( '{vs_i}{nnna}{vs_ai}' or
+     '{vs_ai}' (test not ('{ka}' or '{ca}' or '{tta}' or '{tha}' or '{pa}' or '{rra}'))) or
+      ( '{vs_ai}' (test (('{ka}' or '{ca}' or '{tta}' or '{tha}' or '{pa}' or '{rra}') '{pulli}')))
+    ] <- '{pulli}'
+   
+    )
+   or
+   test ( [ '{vs_o}{tta}{vs_u}' or '{vs_oo}{tta}{vs_u}' or
+     '{vs_i}{la}{pulli}' or '{vs_i}{rra}{pulli}' or
+        ( '{vs_i}{nnna}{pulli}' (test not '{ma}')) or
+      '{vs_i}{nnna}{pulli}{rra}{vs_u}' or
+      '{vs_i}{ra}{vs_u}{na}{pulli}{ta}{vs_u}' or
+      '{va}{vs_i}{tta}' or ($length >= 7 '{vs_i}{tta}{ma}{pulli}') or
+      '{vs_aa}{la}{pulli}' or '{vs_u}{tta}{vs_ai}' or
+      '{vs_aa}{ma}{la}{pulli}' or
+      ( '{la}{pulli}' (test not ('{vs_aa}' or '{vs_i}' or '{vs_ii}' or '{vs_e}' or '{vs_ee}' or '{vs_u}' or '{vs_uu}' or '{vs_ai}'))) or
+      '{vs_u}{lla}{pulli}' ] <- '{pulli}'
+   
+     )
+   or
+   test ( [ '{ka}{nna}{pulli}' or '{ma}{vs_u}{nnna}{pulli}' or
+     '{ma}{vs_ee}{la}{pulli}' or '{ma}{vs_ee}{rra}{pulli}' or
+     '{ka}{vs_ii}{llla}{pulli}' or '{pa}{vs_i}{nnna}{pulli}' or
+     ('{ta}{vs_u}' (test not ('{vs_aa}' or '{vs_i}' or '{vs_ii}' or '{vs_e}' or '{vs_ee}' or '{vs_u}' or '{vs_uu}' or '{vs_ai}'))) ] delete
+   
+      )
+   or
+   test ( [ '{vs_ii}' ] <- '{vs_i}'
+   
+      )
+  )
+  (set found_a_match)
+  (set found_vetrumai_urupu)
+  do ( [ '{vs_i}{nnna}{pulli}' ] <- '{pulli}' )
+ )
+ do fix_endings
+)
+define remove_tense_suffixes as (
+ set found_a_match
+ repeat ( found_a_match (do remove_tense_suffix) )
+)
+define remove_tense_suffix as (
+ unset found_a_match
+ has_min_length
+ backwards (
+  do (
+   test ( ['{ka}{vs_o}{nna}{pulli}{tta}{vs_i}{ra}{pulli}' or
+     '{pa}{tta}{vs_u}']
+     delete
+     (set found_a_match)
+     )
+   or
+   test ( [
+     '{ma}{vs_aa}{ra}{pulli}' or '{ma}{vs_i}{nnna}{pulli}' or
+     '{nnna}{nnna}{pulli}' or '{nnna}{vs_aa}{nnna}{pulli}' or
+     '{nnna}{vs_aa}{lla}{pulli}' or '{nnna}{vs_aa}{ra}{pulli}' or
+     ('{va}{nnna}{pulli}' test (not ( '{a}' or '{aa}' or '{i}' or '{ii}' or '{u}' or '{uu}' or '{e}' or '{ee}' or '{ai}' or '{o}' or '{oo}' or '{au}' )) ) or
+     '{nnna}{lla}{pulli}' or '{va}{lla}{pulli}' or
+     '{nnna}{ra}{pulli}' or '{va}{ra}{pulli}' or
+     '{nnna}' or '{pa}' or '{ka}' or '{ta}' or '{ya}' or
+     '{pa}{nnna}{pulli}' or '{pa}{lla}{pulli}' or
+     '{pa}{ra}{pulli}' or
+     ('{ta}{vs_u}' (test not ('{vs_aa}' or '{vs_i}' or '{vs_ii}' or '{vs_e}' or '{vs_ee}' or '{vs_u}' or '{vs_uu}' or '{vs_ai}'))) or
+     '{vs_i}{rra}{pulli}{rra}{vs_u}' or
+     '{pa}{ma}{pulli}' or '{nnna}{ma}{pulli}' or
+     '{ta}{vs_u}{ma}{pulli}' or '{rra}{vs_u}{ma}{pulli}' or
+     '{ka}{vs_u}{ma}{pulli}' or '{nnna}{vs_e}{nnna}{pulli}' or
+     '{nnna}{vs_ai}' or '{va}{vs_ai}'
+       ] delete
+       (set found_a_match)
+       )
+   or
+   test ( [('{vs_aa}{nnna}{pulli}' test (not '{ca}')) or
+     '{vs_aa}{lla}{pulli}' or
+       '{vs_aa}{ra}{pulli}' or
+        '{vs_ee}{nnna}{pulli}' or '{vs_aa}' or
+        '{vs_aa}{ma}{pulli}' or '{vs_e}{ma}{pulli}' or
+        '{vs_ee}{ma}{pulli}' or '{vs_oo}{ma}{pulli}' or
+        '{ka}{vs_u}{ma}{pulli}' or '{ta}{vs_u}{ma}{pulli}' or
+        '{tta}{vs_u}{ma}{pulli}' or '{rra}{vs_u}{ma}{pulli}' or
+        '{vs_aa}{ya}{pulli}' or '{nnna}{vs_e}{nnna}{pulli}' or
+        '{nnna}{vs_i}{ra}{pulli}' or '{vs_ii}{ra}{pulli}' or
+        '{vs_ii}{ya}{ra}{pulli}' ] <- '{pulli}'
+      (set found_a_match)
+      )
+   or
+   test ( ([ '{ka}{vs_u}' or '{ta}{vs_u}' ) (test '{pulli}') ] delete
+      (set found_a_match)
+      )
+  )
+  do (['{vs_aa}{na}{vs_i}{nnna}{pulli}{rra}' or
+    '{vs_aa}{na}{vs_i}{nnna}{pulli}{rra}{pulli}' or
+    '{ka}{vs_i}{nnna}{pulli}{rra}' or
+    '{ka}{vs_i}{nnna}{pulli}{rra}{pulli}' or
+    '{ka}{vs_i}{rra}' or
+    '{ka}{vs_i}{rra}{pulli}' ] delete
+    (set found_a_match)
+    )
+ )
+ do fix_endings
+)
+define stem as (
+
+ do fix_ending
+ has_min_length
+ do remove_question_prefixes
+ do remove_pronoun_prefixes
+ do remove_question_suffixes
+ do remove_um
+ do remove_common_word_endings
+ do remove_vetrumai_urupukal
+ do remove_plural_suffix
+ do remove_command_suffixes
+ do remove_tense_suffixes
+)
Index: GNUmakefile
===================================================================
--- GNUmakefile	(revision 577)
+++ GNUmakefile	(working copy)
@@ -7,7 +7,7 @@
 libstemmer_algorithms = danish dutch english finnish french german hungarian \
 			italian \
 			norwegian porter portuguese romanian \
-			russian spanish swedish turkish
+			russian spanish swedish tamil turkish
 
 KOI8_R_algorithms = russian
 ISO_8859_1_algorithms = danish dutch english finnish french german hungarian \
Index: libstemmer/modules.txt
===================================================================
--- libstemmer/modules.txt	(revision 577)
+++ libstemmer/modules.txt	(working copy)
@@ -24,6 +24,7 @@
 spanish         UTF_8,ISO_8859_1        spanish,es,esl,spa
 swedish         UTF_8,ISO_8859_1        swedish,sv,swe
 turkish         UTF_8                   turkish,tr,tur
+tamil           UTF_8                   tamil,ta,tam
 
 # Also include the traditional porter algorithm for english.
 # The porter algorithm is included in the libstemmer distribution to assist
Index: libstemmer/modules_utf8.txt
===================================================================
--- libstemmer/modules_utf8.txt	(revision 577)
+++ libstemmer/modules_utf8.txt	(working copy)
@@ -23,6 +23,7 @@
 spanish         UTF_8                   spanish,es,esl,spa
 swedish         UTF_8                   swedish,sv,swe
 turkish         UTF_8                   turkish,tr,tur
+tamil           UTF_8                   tamil,ta,tam
 
 # Also include the traditional porter algorithm for english.
 # The porter algorithm is included in the libstemmer distribution to assist