PhraseQuery against title field and QueryParser against a catch all field do not result in the results I expect

Question

If a user enters a phrase in the search box (with or without quotes), I want the results that show first be the documents that have the exact phrase in the document title, and other documents showing after it. This is what I have tried but it fails to give me the search results in that order:

During indexing I say:

AddStringFieldToDocument(document, "keyWord", this.BuildKeywordsString(), Field.Store.NO, Field.Index.ANALYZED, false);

AddStringFieldToDocument(document, "title", this.Title, Field.Store.NO, Field.Index.ANALYZED, false, 4f);

    private void AddStringFieldToDocument(Document document, string fieldName, string fieldValue, 
        Field.Store store, Field.Index index, bool setOmitTermFreqAndPositions)
    {
        if (fieldValue == null)
        {
            return;
        }
        var field = GetFieldToAddToDocument(document, fieldName, fieldValue, store, index, setOmitTermFreqAndPositions);
        document.Add(field);
    }

    private void AddStringFieldToDocument(Document document, string fieldName, string fieldValue,
        Field.Store store, Field.Index index, bool setOmitTermFreqAndPositions, Single boost)
    {
        if (fieldValue == null)
        {
            return;
        }

        var field = GetFieldToAddToDocument(document, fieldName, fieldValue, store, index, setOmitTermFreqAndPositions);
        field.SetBoost(boost); //boosting title
        document.Add(field);
    }

    private Field GetFieldToAddToDocument(Document document, string fieldName, string fieldValue, Field.Store store,
        Field.Index index, bool setOmitTermFreqAndPositions)
    {
        Field field = new Field(fieldName, fieldValue, store, index);
        field.SetOmitTermFreqAndPositions(setOmitTermFreqAndPositions);
        return field;
    }

At search time as part of my BooleanQuery I have:

if (!string.IsNullOrWhiteSpace(queryString))
        {
            QueryParser qpKeyWord = new QueryParser(myVersionUsed, "keyWord", StandardAnalyzer);

            Query qKeyWord = qpKeyWord.Parse(queryString);
            booleanQuery.Add(qKeyWord, BooleanClause.Occur.MUST);

            Term titleTerm = new Term("title", queryString);
            PhraseQuery qTitleWord = new PhraseQuery();
            qTitleWord.SetSlop(12);
            qTitleWord.Add(titleTerm);
            qTitleWord.SetBoost(5);
            booleanQuery.Add(qTitleWord, BooleanClause.Occur.SHOULD);

The results I get are mixed. Further, when I run IndexSearcher.Explain(query, docId)

I get:

Document Id: 92871
0.5439626 = (MATCH) product of:
  0.8159439 = (MATCH) sum of:
    0.5884751 = (MATCH) sum of:
      0.2580064 = (MATCH) weight(KeyWord:chicken in 92871), product of:
        0.2226703 = queryWeight(KeyWord:chicken), product of:
          3.236447 = idf(docFreq=25345, maxDocs=237239)
          0.06880084 = queryNorm
        1.158692 = (MATCH) fieldWeight(KeyWord:chicken in 92871), product of:
          4.582576 = tf(termFreq(KeyWord:chicken)=21)
          3.236447 = idf(docFreq=25345, maxDocs=237239)
          0.078125 = fieldNorm(field=KeyWord, doc=92871)
      0.3304687 = (MATCH) weight(KeyWord:parmesan in 92871), product of:
        0.2962231 = queryWeight(KeyWord:parmesan), product of:
          4.305515 = idf(docFreq=8701, maxDocs=237239)
          0.06880084 = queryNorm
        1.115608 = (MATCH) fieldWeight(KeyWord:parmesan in 92871), product of:
          3.316625 = tf(termFreq(KeyWord:parmesan)=11)
          4.305515 = idf(docFreq=8701, maxDocs=237239)
          0.078125 = fieldNorm(field=KeyWord, doc=92871)
    0.2274688 = (MATCH) weight(has_photo:y in 92871), product of:
      0.1251001 = queryWeight(has_photo:y), product of:
        1.818294 = idf(docFreq=104665, maxDocs=237239)
        0.06880084 = queryNorm
      1.818294 = (MATCH) fieldWeight(has_photo:y in 92871), product of:
        1 = tf(termFreq(has_photo:y)=1)
        1.818294 = idf(docFreq=104665, maxDocs=237239)
        1 = fieldNorm(field=has_photo, doc=92871)
  0.6666667 = coord(2/3)

Which has no number associated with the PhraseQuery but has separate numbers for each keyWord.
However at search time when I run query.ToString() I get:

+(KeyWord:chicken KeyWord:parmesan) title:"Chicken Parmesan"~12^5.0 

which means that The query was written right. Right? What am I missing?

Answer 1

The way you build up your query for the title I suspect you never get hits originating from the title clause.

You build the PhraseQuery to look for a single term: "Chicken Parmesan", but when you indexed it, the StandardAnalyzer produced two Terms: "chicken" and "parmesan". You need to build the PhraseQuery with those two terms.

You can use the QueryParser for this purpose:

    QueryParser qp = new QueryParser("keyWord", new StandardAnalyzer());
    Query q = qp.Parse("+(keyWord:chicken KeyWord:parmesan) title:\"Chicken Parmesan\"~12^5.0");
    var hits = searcher.Search(q);

If you dont want to use the QueryParser, use the TokenStream api to break your text in tokens:

    PhraseQuery titleQuery = new PhraseQuery();
    titleQuery.SetSlop(12);
    titleQuery.SetBoost(5);
    BooleanQuery keywordQuery = new BooleanQuery();

    var standard = new StandardAnalyzer();
    TokenStream tokens = standard.TokenStream("title", new StringReader("Chicken Parmesan"));
    List<Term> terms = new List<Term>();
    while (tokens.IncrementToken())
    {
        TermAttribute termAttribute = (TermAttribute)tokens.GetAttribute(typeof(TermAttribute));

        titleQuery.Add(new Term("title", termAttribute.Term()));

        keywordQuery.Add(
            new TermQuery(
                new Term("keyWord", termAttribute.Term())),
            BooleanClause.Occur.SHOULD);  
    }

    BooleanQuery query = new BooleanQuery();
    query.Add(keywordQuery, BooleanClause.Occur.MUST);
    query.Add(titleQuery, BooleanClause.Occur.SHOULD);

    var hits = searcher.Search(query);