An updated and ready-to-use version of the code proposed above.
This code is compatible with Apache Lucene 5.x…6.x.
CardKeyword class:
import java.util.HashSet;
import java.util.Set;
/**
* Keyword card with stem form, terms dictionary and frequency rank
*/
class CardKeyword implements Comparable {
/**
* Stem form of the keyword
*/
private final String stem;
/**
* Terms dictionary
*/
private final Set terms = new HashSet<>();
/**
* Frequency rank
*/
private int frequency;
/**
* Build keyword card with stem form
*
* @param stem
*/
public CardKeyword(String stem) {
this.stem = stem;
}
/**
* Add term to the dictionary and update its frequency rank
*
* @param term
*/
public void add(String term) {
this.terms.add(term);
this.frequency++;
}
/**
* Compare two keywords by frequency rank
*
* @param keyword
* @return int, which contains comparison results
*/
@Override
public int compareTo(CardKeyword keyword) {
return Integer.valueOf(keyword.frequency).compareTo(this.frequency);
}
/**
* Get stem's hashcode
*
* @return int, which contains stem's hashcode
*/
@Override
public int hashCode() {
return this.getStem().hashCode();
}
/**
* Check if two stems are equal
*
* @param o
* @return boolean, true if two stems are equal
*/
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof CardKeyword)) return false;
CardKeyword that = (CardKeyword) o;
return this.getStem().equals(that.getStem());
}
/**
* Get stem form of keyword
*
* @return String, which contains getStemForm form
*/
public String getStem() {
return this.stem;
}
/**
* Get terms dictionary of the stem
*
* @return Set, which contains set of terms of the getStemForm
*/
public Set getTerms() {
return this.terms;
}
/**
* Get stem frequency rank
*
* @return int, which contains getStemForm frequency
*/
public int getFrequency() {
return this.frequency;
}
}
KeywordsExtractor class:
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
import org.apache.lucene.analysis.standard.ClassicFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.IOException;
import java.io.StringReader;
import java.util.*;
/**
* Keywords extractor functionality handler
*/
class KeywordsExtractor {
/**
* Get list of keywords with stem form, frequency rank, and terms dictionary
*
* @param fullText
* @return List, which contains keywords cards
* @throws IOException
*/
static List getKeywordsList(String fullText) throws IOException {
TokenStream tokenStream = null;
try {
// treat the dashed words, don't let separate them during the processing
fullText = fullText.replaceAll("-+", "-0");
// replace any punctuation char but apostrophes and dashes with a space
fullText = fullText.replaceAll("[\\p{Punct}&&[^'-]]+", " ");
// replace most common English contractions
fullText = fullText.replaceAll("(?:'(?:[tdsm]|[vr]e|ll))+\\b", "");
StandardTokenizer stdToken = new StandardTokenizer();
stdToken.setReader(new StringReader(fullText));
tokenStream = new StopFilter(new ASCIIFoldingFilter(new ClassicFilter(new LowerCaseFilter(stdToken))), EnglishAnalyzer.getDefaultStopSet());
tokenStream.reset();
List cardKeywords = new LinkedList<>();
CharTermAttribute token = tokenStream.getAttribute(CharTermAttribute.class);
while (tokenStream.incrementToken()) {
String term = token.toString();
String stem = getStemForm(term);
if (stem != null) {
CardKeyword cardKeyword = find(cardKeywords, new CardKeyword(stem.replaceAll("-0", "-")));
// treat the dashed words back, let look them pretty
cardKeyword.add(term.replaceAll("-0", "-"));
}
}
// reverse sort by frequency
Collections.sort(cardKeywords);
return cardKeywords;
} finally {
if (tokenStream != null) {
try {
tokenStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
* Get stem form of the term
*
* @param term
* @return String, which contains the stemmed form of the term
* @throws IOException
*/
private static String getStemForm(String term) throws IOException {
TokenStream tokenStream = null;
try {
StandardTokenizer stdToken = new StandardTokenizer();
stdToken.setReader(new StringReader(term));
tokenStream = new PorterStemFilter(stdToken);
tokenStream.reset();
// eliminate duplicate tokens by adding them to a set
Set stems = new HashSet<>();
CharTermAttribute token = tokenStream.getAttribute(CharTermAttribute.class);
while (tokenStream.incrementToken()) {
stems.add(token.toString());
}
// if stem form was not found or more than 2 stems have been found, return null
if (stems.size() != 1) {
return null;
}
String stem = stems.iterator().next();
// if the stem form has non-alphanumerical chars, return null
if (!stem.matches("[a-zA-Z0-9-]+")) {
return null;
}
return stem;
} finally {
if (tokenStream != null) {
try {
tokenStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
* Find sample in collection
*
* @param collection
* @param sample
* @param
* @return T, which contains the found object within collection if exists, otherwise the initially searched object
*/
private static T find(Collection collection, T sample) {
for (T element : collection) {
if (element.equals(sample)) {
return element;
}
}
collection.add(sample);
return sample;
}
}
The call of function:
String text = "…";
List keywordsList = KeywordsExtractor.getKeywordsList(text);
