Module org.apache.lucene.core

Package org.apache.lucene.codecs.lucene95

Class Lucene95HnswVectorsWriter

java.lang.Object

org.apache.lucene.codecs.KnnVectorsWriter

org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsWriter

All Implemented Interfaces:

Closeable, AutoCloseable, Accountable

public final class Lucene95HnswVectorsWriter
extends KnnVectorsWriter

Writes vector values and knn graphs to index segments.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
private static class	Lucene95HnswVectorsWriter.FieldWriter<T>
private static class	Lucene95HnswVectorsWriter.RAVectorValues<T>

Nested classes/interfaces inherited from class org.apache.lucene.codecs.KnnVectorsWriter

KnnVectorsWriter.MergedVectorValues

Field Summary

Fields

Modifier and Type	Field	Description
private final int	beamWidth
private final List<Lucene95HnswVectorsWriter.FieldWriter<?>>	fields
private boolean	finished
private final int	M
private final IndexOutput	meta
private final SegmentWriteState	segmentWriteState
private final IndexOutput	vectorData
private final IndexOutput	vectorIndex

Fields inherited from interface org.apache.lucene.util.Accountable

NULL_ACCOUNTABLE

Constructor Summary

Constructors

Constructor	Description
Lucene95HnswVectorsWriter(SegmentWriteState state, int M, int beamWidth)

Method Summary

Modifier and Type	Method	Description
KnnFieldVectorsWriter<?>	addField(FieldInfo fieldInfo)	Add new field for indexing
void	close()
void	finish()	Called once at the end before close
void	flush(int maxDoc, Sorter.DocMap sortMap)	Flush all buffered data on disk *
void	mergeOneField(FieldInfo fieldInfo, MergeState mergeState)	Write field for merging
long	ramBytesUsed()	Return the memory usage of this object in bytes.
private HnswGraph	reconstructAndWriteGraph(OnHeapHnswGraph graph, int[] newToOldMap, int[] oldToNewMap, int[][] levelNodeOffsets)	Reconstructs the graph given the old and new node ids.
private void	reconstructAndWriteNeigbours(NeighborArray neighbors, int[] oldToNewMap, int maxConnOnLevel, int maxOrd)
private static DocsWithFieldSet	writeByteVectorData(IndexOutput output, ByteVectorValues byteVectorValues)	Writes the byte vector values to the output and returns a set of documents that contains vectors.
private void	writeByteVectors(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData)
private void	writeField(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData, int maxDoc)
private void	writeFloat32Vectors(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData)
private int[][]	writeGraph(OnHeapHnswGraph graph)
private void	writeMeta(FieldInfo field, int maxDoc, long vectorDataOffset, long vectorDataLength, long vectorIndexOffset, long vectorIndexLength, DocsWithFieldSet docsWithField, HnswGraph graph, int[][] graphLevelNodeOffsets)
private long	writeSortedByteVectors(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData, int[] ordMap)
private long	writeSortedFloat32Vectors(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData, int[] ordMap)
private void	writeSortingField(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData, int maxDoc, Sorter.DocMap sortMap)
private static DocsWithFieldSet	writeVectorData(IndexOutput output, FloatVectorValues floatVectorValues)	Writes the vector values to the output and returns a set of documents that contains vectors.

Methods inherited from class org.apache.lucene.codecs.KnnVectorsWriter

merge

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.lucene.util.Accountable

getChildResources

Field Details

segmentWriteState

private final SegmentWriteState segmentWriteState

meta

private final IndexOutput meta

vectorData

private final IndexOutput vectorData

vectorIndex

private final IndexOutput vectorIndex

M

private final int M

beamWidth

private final int beamWidth

fields

private final List<Lucene95HnswVectorsWriter.FieldWriter<?>> fields

finished

private boolean finished

Constructor Details

Lucene95HnswVectorsWriter

Lucene95HnswVectorsWriter(SegmentWriteState state,
 int M,
 int beamWidth)
                   throws IOException

Throws:

IOException

Method Details

addField

public KnnFieldVectorsWriter<?> addField(FieldInfo fieldInfo)
                                  throws IOException

Description copied from class: KnnVectorsWriter

Add new field for indexing

Specified by:

addField in class KnnVectorsWriter

Throws:

IOException

flush

public void flush(int maxDoc,
 Sorter.DocMap sortMap)
           throws IOException

Description copied from class: KnnVectorsWriter

Flush all buffered data on disk *

Specified by:

flush in class KnnVectorsWriter

Throws:

IOException

finish

public void finish()
            throws IOException

Description copied from class: KnnVectorsWriter

Called once at the end before close

Specified by:

finish in class KnnVectorsWriter

Throws:

IOException

ramBytesUsed

public long ramBytesUsed()

Description copied from interface: Accountable

Return the memory usage of this object in bytes. Negative values are illegal.

writeField

private void writeField(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData,
 int maxDoc)
                 throws IOException

Throws:

IOException

writeFloat32Vectors

private void writeFloat32Vectors(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData)
                          throws IOException

Throws:

IOException

writeByteVectors

private void writeByteVectors(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData)
                       throws IOException

Throws:

IOException

writeSortingField

private void writeSortingField(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData,
 int maxDoc,
 Sorter.DocMap sortMap)
                        throws IOException

Throws:

IOException

writeSortedFloat32Vectors

private long writeSortedFloat32Vectors(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData,
 int[] ordMap)
                                throws IOException

Throws:

IOException

writeSortedByteVectors

private long writeSortedByteVectors(Lucene95HnswVectorsWriter.FieldWriter<?> fieldData,
 int[] ordMap)
                             throws IOException

Throws:

IOException

reconstructAndWriteGraph

private HnswGraph reconstructAndWriteGraph(OnHeapHnswGraph graph,
 int[] newToOldMap,
 int[] oldToNewMap,
 int[][] levelNodeOffsets)
                                    throws IOException

Reconstructs the graph given the old and new node ids.

Additionally, the graph node connections are written to the vectorIndex.

Parameters:

graph - The current on heap graph

newToOldMap - the new node ids indexed to the old node ids

oldToNewMap - the old node ids indexed to the new node ids

levelNodeOffsets - where to place the new offsets for the nodes in the vector index.

Returns:

The graph

Throws:

IOException - if writing to vectorIndex fails

reconstructAndWriteNeigbours

private void reconstructAndWriteNeigbours(NeighborArray neighbors,
 int[] oldToNewMap,
 int maxConnOnLevel,
 int maxOrd)
                                   throws IOException

Throws:

IOException

mergeOneField

public void mergeOneField(FieldInfo fieldInfo,
 MergeState mergeState)
                   throws IOException

Description copied from class: KnnVectorsWriter

Write field for merging

Overrides:

mergeOneField in class KnnVectorsWriter

Throws:

IOException

writeGraph

private int[][] writeGraph(OnHeapHnswGraph graph)
                    throws IOException

Parameters:

graph - Write the graph in a compressed format

Returns:

The non-cumulative offsets for the nodes. Should be used to create cumulative offsets.

Throws:

IOException - if writing to vectorIndex fails

writeMeta

private void writeMeta(FieldInfo field,
 int maxDoc,
 long vectorDataOffset,
 long vectorDataLength,
 long vectorIndexOffset,
 long vectorIndexLength,
 DocsWithFieldSet docsWithField,
 HnswGraph graph,
 int[][] graphLevelNodeOffsets)
                throws IOException

Throws:

IOException

writeByteVectorData

private static DocsWithFieldSet writeByteVectorData(IndexOutput output,
 ByteVectorValues byteVectorValues)
                                             throws IOException

Writes the byte vector values to the output and returns a set of documents that contains vectors.

Throws:

IOException

writeVectorData

private static DocsWithFieldSet writeVectorData(IndexOutput output,
 FloatVectorValues floatVectorValues)
                                         throws IOException

Writes the vector values to the output and returns a set of documents that contains vectors.

Throws:

IOException

close

public void close()
           throws IOException

Throws:

IOException