Searching with Lucene

String tokenRegex = /(?ix)             # ignore case, enable whitespace & comments
    \b                                 # word boundary
    (                                  # start capture of all terms
        (                              # capture project name term
            (apache|eclipse)\s         # foundation name
            (commons\s)?               # optional subproject name
            (
                ?!(groovy              # negative lookahead for excluded words
                | and   | license  | users
                | https | projects | software
                | or    | prefixes | technologies)
            )\w+
        )                              # end capture project name term
        |                              # alternatively
        (                              # capture non-project term
            \w+?\b                     # non-greedily match any other word chars
        )                              # end capture non-project term
    )                                  # end capture term
/

String baseDir = '/projects/groovy-website/site/src/site/blog' // (1)

var histogram = [:].withDefault { 0 } // (1)

new File(baseDir).traverse(nameFilter: ~/.*\.adoc/) { file ->  // (2)
    var m = file.text =~ tokenRegex // (3)
    var projects = m*.get(2).grep()*.toLowerCase()*.replaceAll('\n', ' ') // (4)
    var counts = projects.countBy() // (5)
    if (counts) {
        println "$file.name: $counts" // (6)
        counts.each { k, v -> histogram[k] += v } // (7)
    }
}

println "\nFrequency of total hits mentioning a project:"
histogram.sort { e -> -e.value }.each { k, v -> // (8)
    var label = "$k ($v)"
    println "${label.padRight(32)} ${bar(v, 0, 50, 50)}"
}

class ProjectNameAnalyzer extends Analyzer {
    @Override
    protected TokenStreamComponents createComponents(String fieldName) {
        var src = new PatternTokenizer(~tokenRegex, 0)
        var result = new LowerCaseFilter(src)
        new TokenStreamComponents(src, result)
    }
}

var analyzer = new ProjectNameAnalyzer() // (1)
var indexDir = new ByteBuffersDirectory() // (2)
var config = new IndexWriterConfig(analyzer)

new IndexWriter(indexDir, config).withCloseable { writer ->
    var indexedWithFreq = new FieldType(stored: true,
        indexOptions: IndexOptions.DOCS_AND_FREQS,
        storeTermVectors: true)
    new File(baseDir).traverse(nameFilter: ~/.*\.adoc/) { file ->
        file.withReader { br ->
            var document = new Document()
            document.add(new Field('content', br.text, indexedWithFreq)) // (3)
            document.add(new StringField('name', file.name, Field.Store.YES)) // (4)
            writer.addDocument(document)
        }
    }
}

var reader = DirectoryReader.open(indexDir)
var vectors = reader.termVectors()
var storedFields = reader.storedFields()

Set projects = []
for (docId in 0..<reader.maxDoc()) {
    String name = storedFields.document(docId).get('name')
    TermsEnum terms = vectors.get(docId, 'content').iterator() // (1)
    var found = [:]
    while (terms.next() != null) {
        PostingsEnum postingsEnum = terms.postings(null, PostingsEnum.ALL)
        while (postingsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
            int freq = postingsEnum.freq()
            var string = terms.term().utf8ToString().replaceAll('\n', ' ')
            if (string.startsWith('apache ') || string.startsWith('eclipse ')) { // (2)
                found[string] = freq
            }
        }
    }
    if (found) {
        println "$name: $found"
        projects += found.keySet()
    }
}

var terms = projects.collect { name -> new Term('content', name) }
var byReverseValue = { e -> -e.value }

println "\nFrequency of total hits mentioning a project (top 10):"
var termFreq = terms.collectEntries { term ->
    [term.text(), reader.totalTermFreq(term)]  // (3)
}
termFreq.sort(byReverseValue).take(10).each { k, v ->
    var label = "$k ($v)"
    println "${label.padRight(32)} ${bar(v, 0, 50, 50)}"
}

println "\nFrequency of documents mentioning a project (top 10):"
var docFreq = terms.collectEntries { term ->
    [term.text(), reader.docFreq(term)]  // (4)
}
docFreq.sort(byReverseValue).take(10).each { k, v ->
    var label = "$k ($v)"
    println "${label.padRight(32)} ${bar(v * 2, 0, 20, 20)}"
}

var parser = new QueryParser("content", analyzer)
var searcher = new IndexSearcher(reader)
var query = parser.parse('emoji*')
var results = searcher.search(query, 10)
println "\nTotal documents with hits for $query --> $results.totalHits"
results.scoreDocs.each {
    var doc = storedFields.document(it.doc)
    println "${doc.get('name')}"
}

Total documents with hits for content:emoji* --> 11 hits
adventures-with-groovyfx.adoc
create-groovy-blog.adoc
deep-learning-and-eclipse-collections.adoc
fruity-eclipse-collections.adoc
groovy-haiku-processing.adoc
groovy-lucene.adoc
helloworldemoji.adoc
seasons-greetings-emoji.adoc
set-operations-with-groovy.adoc
solving-simple-optimization-problems-with-groovy.adoc

new IndexWriter(indexDir, config).withCloseable { writer ->
    new File(baseDir).traverse(nameFilter: ~/.*\.adoc/) { file ->
        file.withReader { br ->
            var document = new Document()
            var fieldType = new FieldType(stored: true,
                indexOptions: IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
                storeTermVectors: true,
                storeTermVectorPositions: true,
                storeTermVectorOffsets: true)
            document.add(new Field('content', br.text, fieldType))
            document.add(new StringField('name', file.name, Field.Store.YES))
            writer.addDocument(document)
        }
    }
}

List<String> handleHit(ScoreDoc hit, Query query, DirectoryReader dirReader) {
    boolean phraseHighlight = true
    boolean fieldMatch = true
    var fieldQuery = new FieldQuery(query, dirReader, phraseHighlight, fieldMatch)
    var stack = new FieldTermStack(dirReader, hit.doc, 'content', fieldQuery)
    var phrases = new FieldPhraseList(stack, fieldQuery)
    phrases.phraseList*.termsInfos*.text.flatten() // (1)
}

var query = parser.parse(/apache\ * OR eclipse\ */) // (1)
var results = searcher.search(query, 30) // (2)
println "Total documents with hits for $query --> $results.totalHits\n"

var storedFields = searcher.storedFields()
var histogram = [:].withDefault { 0 }
results.scoreDocs.each { ScoreDoc scoreDoc -> // (3)
    var doc = storedFields.document(scoreDoc.doc)
    var found = handleHit(scoreDoc, query, reader) // (4)
    println "${doc.get('name')}: ${found*.replaceAll('\n', ' ').countBy()}"
    found.each { histogram[it.replaceAll('\n', ' ')] += 1 } // (5)
}

println "\nFrequency of total hits mentioning a project (top 10):"
histogram.sort { e -> -e.value }.take(10).each { k, v -> // (6)
    var label = "$k ($v)"
    println "${label.padRight(32)} ${bar(v, 0, 50, 50)}"
}

var analyzer = new ProjectNameAnalyzer()
var indexDir = new ByteBuffersDirectory()
var taxonDir = new ByteBuffersDirectory()
var config = new IndexWriterConfig(analyzer)
var indexWriter = new IndexWriter(indexDir, config) // (1)
var taxonWriter = new DirectoryTaxonomyWriter(taxonDir) // (2)

var fConfig = new FacetsConfig().tap { // (3)
    setHierarchical('projectNameCounts', true)
    setMultiValued('projectNameCounts', true)
    setMultiValued('projectFileCounts', true)
    setMultiValued('projectHitCounts', true)
    setIndexFieldName('projectHitCounts', '$projectHitCounts')
}

new File(baseDir).traverse(nameFilter: ~/.*\.adoc/) { file ->
    var m = file.text =~ tokenRegex
    var projects = m*.get(2).grep()*.toLowerCase()*.replaceAll('\n', ' ').countBy()
    file.withReader { br ->
        var document = new Document()
        var indexedWithFreq = new FieldType(stored: true,
            indexOptions: IndexOptions.DOCS_AND_FREQS,
            storeTermVectors: true)
        document.add(new Field('content', br.text, indexedWithFreq))
        document.add(new StringField('name', file.name, Field.Store.YES))
        if (projects) {
            println "$file.name: $projects"
            projects.each { k, v -> // (4)
                document.add(new IntAssociationFacetField(v, 'projectHitCounts', k))
                document.add(new FacetField('projectFileCounts', k))
                document.add(new FacetField('projectNameCounts', k.split()))
            }
        }
        indexWriter.addDocument(fConfig.build(taxonWriter, document))
    }
}
indexWriter.close()
taxonWriter.close()

var reader = DirectoryReader.open(indexDir)
var searcher = new IndexSearcher(reader)
var taxonReader = new DirectoryTaxonomyReader(taxonDir)
var fcm = new FacetsCollectorManager()
var query = new MatchAllDocsQuery()
var fc = FacetsCollectorManager.search(searcher, query, 0, fcm).facetsCollector()

var topN = 5
var projects = new TaxonomyFacetIntAssociations('$projectHitCounts', taxonReader,
    fConfig, fc, AssociationAggregationFunction.SUM)
var hitData = projects.getTopChildren(topN, 'projectHitCounts').labelValues

println "\nFrequency of total hits mentioning a project (top $topN):"
hitData.each { m ->
    var label = "$m.label ($m.value)"
    println "${label.padRight(32)} ${bar(m.value, 0, 50, 50)}"
}

println "\nFrequency of documents mentioning a project (top $topN):"
hitData.each { m ->
    var label = "$m.label ($m.count)"
    println "${label.padRight(32)} ${bar(m.count * 2, 0, 20, 20)}"
}

var facets = new FastTaxonomyFacetCounts(taxonReader, fConfig, fc)

println "\nFrequency of documents mentioning a project (top $topN):"
println facets.getTopChildren(topN, 'projectFileCounts')

['apache', 'commons'].inits().reverseEach { path -> // (1)
    println "Frequency of documents mentioning a project with path $path (top $topN):"
    println "${facets.getTopChildren(topN, 'projectNameCounts', *path)}"
}

var parser = new QueryParser('content', analyzer)
var query = parser.parse(/apache\ * AND eclipse\ * AND emoji*/)
var results = searcher.search(query, topN)
var storedFields = searcher.storedFields()
assert results.totalHits.value() == 1 &&
    storedFields.document(results.scoreDocs[0].doc).get('name') == 'fruity-eclipse-collections.adoc'

var analyzer = new StandardAnalyzer()
var indexDir = new ByteBuffersDirectory()
var config = new IndexWriterConfig(analyzer)

new IndexWriter(indexDir, config).withCloseable { writer ->
    new File(baseDir).traverse(nameFilter: ~/.*\.adoc/) { file ->
        file.withReader { br ->
            var document = new Document()
            var fieldType = new FieldType(stored: true,
              indexOptions: IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
              storeTermVectors: true,
              storeTermVectorPositions: true,
              storeTermVectorOffsets: true)
            document.add(new Field('content', br.text, fieldType))
            document.add(new StringField('name', file.name, Field.Store.YES))
            writer.addDocument(document)
        }
    }
}

IndexReader reader = DirectoryReader.open(indexDir)
var searcher = new IndexSearcher(reader)

var projects = [
    'math', 'spark', 'lucene', 'collections', 'deeplearning4j',
    'beam', 'wayang', 'csv', 'io', 'numbers', 'ignite', 'mxnet', 'age',
    'nlpcraft', 'pekko', 'hugegraph', 'tinkerpop', 'commons',
    'cli', 'opennlp', 'ofbiz', 'codec', 'kie', 'flink'
]
var suffix = new SpanMultiTermQueryWrapper(new RegexpQuery( // (1)
    new Term('content', "(${projects.join('|')})")))

// look for apache commons <suffix>
SpanQuery[] spanTerms = ['apache', 'commons'].collect{
    new SpanTermQuery(new Term('content', it))
} + suffix
var apacheCommons = new SpanNearQuery(spanTerms, 0, true)

// look for (apache|eclipse) <suffix>
var foundation = new SpanMultiTermQueryWrapper(new RegexpQuery(
    new Term('content', '(apache|eclipse)')))
var otherProject = new SpanNearQuery([foundation, suffix] as SpanQuery[], 0, true)

var builder = new BooleanQuery.Builder(minimumNumberShouldMatch: 1)
builder.add(otherProject, BooleanClause.Occur.SHOULD)
builder.add(apacheCommons, BooleanClause.Occur.SHOULD)
var query = builder.build()
var results = searcher.search(query, 30)
println "Total documents with hits for $query --> $results.totalHits"

Total documents with hits for (spanNear([SpanMultiTermQueryWrapper(content:/(apache|eclipse)/), SpanMultiTermQueryWrapper(content:/(math|spark|lucene|collections|deeplearning4j|beam|wayang|csv|io|numbers|ignite|mxnet|age|nlpcraft|pekko|hugegraph|tinkerpop|commons|cli|opennlp|ofbiz|codec|kie|flink)/)], 0, true) spanNear([content:apache, content:commons, SpanMultiTermQueryWrapper(content:/(math|spark|lucene|collections|deeplearning4j|beam|wayang|csv|io|numbers|ignite|mxnet|age|nlpcraft|pekko|hugegraph|tinkerpop|commons|cli|opennlp|ofbiz|codec|kie|flink)/)], 0, true))~1 --> 28 hits

var suffix = "(${projects.join('|')})"
var query = span('apache', 'commons', ~suffix) | span(~'(apache|eclipse)', ~suffix)
var results = searcher.search(query, 30)
println "Total documents with hits for $query --> $results.totalHits"

query = span('jackson', 'databind') | span(~'virt.*', 'threads')
results = searcher.search(query, 30)
println "Total documents with hits for $query --> $results.totalHits"

Total documents with hits for (spanNear([content:jackson, content:databind], 0, true) spanNear([SpanMultiTermQueryWrapper(content:/virt.*/), content:threads], 0, true))~1 --> 8 hits

var vectors = reader.termVectors()
var storedFields = reader.storedFields()

var emojis = [:].withDefault { [] as Set }
for (docId in 0..<reader.maxDoc()) {
    String name = storedFields.document(docId).get('name')
    TermsEnum terms = vectors.get(docId, 'content').iterator()
    while (terms.next() != null) {
        PostingsEnum postingsEnum = terms.postings(null, PostingsEnum.ALL)
        while (postingsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
            var string = terms.term().utf8ToString()
            if (string.codePoints().allMatch(Character::isEmojiPresentation)) {
                emojis[name] += string
            }
        }
    }
}
emojis.collect { k, v -> "$k: ${v.join(', ')}" }.each { println it }