is there is any implementation of stemmers for indian languages like(hindi,telugu) are available ....
Hindi Analyzer, with stemmer, is available in Lucene. It is based on this algorithm(pdf).
hindi_stemmer is a Python implementation of the Hindi stemmer described in "A Lightweight Stemmer for Hindi" by Ananthakrishnan Ramanathan and Durgesh D Rao.
import java.util.Map;
import java.util.WeakHashMap;
/** * Hindi light stemmer- removes number, gender and case suffixes from nouns and adjectives
public class HindiStemmerLight{
/**
* A cache of words and their stems
*/
static private Map<String, String> cache = new WeakHashMap<String, String>();
/**
* A buffer of the current word being stemmed
*/
private StringBuilder sb = new StringBuilder();
/**
* Default constructor
*/
public HindiStemmerLight() {
}
public String stem(String word) {
String result = cache.get(word);
if (result != null)
return result;
//
sb.delete(0, sb.length());
//
sb.append(word);
/* remove the case endings from nouns and adjectives */
remove_suffix(sb);
result = sb.toString();
cache.put(word, result);
return result;
}
private void remove_suffix(StringBuilder word) {
int len = word.length() - 1;
/* article */
if (len > 4) {
if (word.substring( len- 2, len+1).equals("िया")) {
word.delete(len-2 , len + 1);
return;
}
if (word.substring( len- 2, len+1).equals("ियो")) {
word.delete(len-2 , len + 1);
return;
}
} /* end if len >4 */
if (len > 3) {
if (word.substring(len-1, len+1).equals("ाए")) {
word.delete(len - 1, len + 1);
return;
}
if (word.substring(len-1, len+1).equals(" ाओ")) {
word.delete(len - 1, len + 1);
return;
}
if (word.substring(len-1, len+1).equals(" ुआ")) {
word.delete(len - 1, len + 1);
return;
}
if (word.substring(len-1, len+1).equals(" ुओ")) {
word.delete(len - 1, len + 1);
return;
}
if (word.substring( len- 1, len+1).equals("ये")) {
word.delete(len-1 , len + 1);
return;
}
if (word.substring(len-1, len+1).equals(" ेन")) {
word.delete(len - 1, len + 1);
return;
}
if (word.substring(len-1, len+1).equals(" ेण")) {
word.delete(len - 1, len + 1);
return;
}
if (word.substring( len- 1, len+1).equals(" ीय")) {
word.delete(len-1 , len + 1);
return;
}
if (word.substring(len-1, len+1).equals("टी")) {
word.delete(len - 1, len + 1);
return;
}
if (word.substring(len-1, len+1).equals("ार")) {
word.delete(len - 1, len + 1);
return;
}
if (word.substring(len-1, len+1).equals("ाई")) {
word.delete(len - 1, len + 1);
return;
}
} /* end if len > 3 */
if (len > 2) {
if (word.substring(len, len+1).equals(" ा")) {
word.delete(len , len + 1);
return;
}
if (word.substring(len, len+1).equals(" े")) {
word.delete(len , len + 1);
return;
}
if (word.substring(len, len+1).equals(" ी")) {
word.delete(len , len + 1);
return;
}
if (word.substring(len, len+1).equals(" ो")) {
word.delete(len , len + 1);
return;
}
if (word.substring(len, len+1).equals("ि ")) {
word.delete(len , len + 1);
return;
}
if (word.substring(len, len+1).equals("अ")) {
word.delete(len , len + 1);
return;
}
} /* end if len > 2 */
return;
}
}
We created a java version of original code Python Hindi Stemmer
Although the way authors of original Hindi stemmer used variable L is not well understood but there is a complete code what would work
import java.util.ArrayList; import org.apache.commons.lang.StringUtils;
public class SimpleHindiStemmer {
/* This is not coming while pasting the code better copy from Python Code as given in suffixes[1] */
private static String [] stem1 = new String [] { "ो", "े", "ू", "ु", "ी", "ि", "ा" };
/* This is not coming while pasting the code better copy from Python Code as given in suffixes[2] */
private static String [] stem2 = new String [] { "कर", "ाओ", "िए", "ाई", "ाए", "ने", "नी", "ना", "ते", "ीं", "ती", "ता", "ाँ", "ां", "ों", "ें" };
private static String [] stem3 = new String [] { "ाकर", "ाइए", "ाईं", "ाया", "ेगी", "ेगा", "ोगी", "ोगे", "ाने", "ाना", "ाते", "ाती", "ाता", "तीं", "ाओं", "ाएं", "ुओं", "ुएं", "ुआं" };
private static String [] stem4 = new String [] { "ाएगी", "ाएगा", "ाओगी", "ाओगे", "एंगी", "ेंगी", "एंगे", "ेंगे", "ूंगी", "ूंगा", "ातीं", "नाओं", "नाएं", "ताओं", "ताएं", "ियाँ", "ियों", "ियां" };
private static String [] stem5 = new String [] { "ाएंगी", "ाएंगे", "ाऊंगी", "ाऊंगा", "ाइयाँ", "ाइयों", "ाइयां" };
private static ArrayList<String []> stemList = new ArrayList<String []>(5);
static int [] cut = new int [] { 0, 1, 1, 1, 2, 2 };
static {
stemList.add(stem5);
stemList.add(stem4);
stemList.add(stem3);
stemList.add(stem2);
stemList.add(stem1);
}
public SimpleHindiStemmer(){
}
public static void main(String [] argv){
SimpleHindiStemmer sm = new SimpleHindiStemmer();
String word = "रास्ते";
System.out.println(sm.stemprocess(word));
}
public String stemprocess(String word){
int wlen = word.length();
int wordlen = wlen*3;
int icnt = 5;
for (String [] stemwords : stemList){
if(wordlen > (icnt + 1)){
for(String sw: stemwords){
if(StringUtils.endsWith(word, sw)){
return StringUtils.substring(word, 0, wlen - cut[icnt]);
}
}
}
icnt--;
}
return word;
}
}
As you see utf-8 chars in some cases are not well captured . Look at original python code and just copy suffixes values from there.
来源：https://stackoverflow.com/questions/4007558/is-there-is-any-stemmer-available-for-indian-language