Skip to content

Create file open hints on IOContext to replace ReadAdvice #14482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -415,7 +415,7 @@ FlushedSegment flush(DocumentsWriter.FlushNotifications flushNotifications) thro
segmentInfo,
fieldInfos.finish(),
pendingUpdates,
new IOContext(new FlushInfo(numDocsInRAM, lastCommittedBytesUsed)));
IOContext.flush(new FlushInfo(numDocsInRAM, lastCommittedBytesUsed)));
final double startMBUsed = lastCommittedBytesUsed / 1024. / 1024.;

// Apply delete-by-docID now (delete-byDocID only
Expand Down Expand Up @@ -599,7 +599,7 @@ void sealFlushedSegment(
IndexWriter.setDiagnostics(newSegment.info, IndexWriter.SOURCE_FLUSH);

IOContext context =
new IOContext(new FlushInfo(newSegment.info.maxDoc(), newSegment.sizeInBytes()));
IOContext.flush(new FlushInfo(newSegment.info.maxDoc(), newSegment.sizeInBytes()));

boolean success = false;
try {
Expand Down
8 changes: 4 additions & 4 deletions lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
Original file line number Diff line number Diff line change
Expand Up @@ -3058,7 +3058,7 @@ public long addIndexes(Directory... dirs) throws IOException {
}

IOContext context =
new IOContext(new FlushInfo(info.info.maxDoc(), info.sizeInBytes()));
IOContext.flush(new FlushInfo(info.info.maxDoc(), info.sizeInBytes()));

FieldInfos fis = readFieldInfos(info);
for (FieldInfo fi : fis) {
Expand Down Expand Up @@ -3400,7 +3400,7 @@ public void addIndexesReaderMerge(MergePolicy.OneMerge merge) throws IOException
testReserveDocs(numDocs);

final IOContext context =
new IOContext(
IOContext.merge(
new MergeInfo(Math.toIntExact(numDocs), -1, false, UNBOUNDED_MAX_MERGE_SEGMENTS));

TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(mergeDirectory);
Expand Down Expand Up @@ -3950,7 +3950,7 @@ public void setMergeInfo(SegmentCommitInfo info) {
boolean closeReaders = true;
try {
for (MergePolicy.OneMerge merge : pointInTimeMerges.merges) {
IOContext context = new IOContext(merge.getStoreMergeInfo());
IOContext context = IOContext.merge(merge.getStoreMergeInfo());
merge.initMergeReaders(
sci -> {
final ReadersAndUpdates rld = getPooledInstance(sci, true);
Expand Down Expand Up @@ -5139,7 +5139,7 @@ private int mergeMiddle(MergePolicy.OneMerge merge, MergePolicy mergePolicy) thr
merge.checkAborted();

Directory mergeDirectory = mergeScheduler.wrapForMerge(merge, directory);
IOContext context = new IOContext(merge.getStoreMergeInfo());
IOContext context = IOContext.merge(merge.getStoreMergeInfo());

final TrackingDirectoryWrapper dirWrapper = new TrackingDirectoryWrapper(mergeDirectory);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ private synchronized void handleDVUpdates(
}
final long nextDocValuesGen = info.getNextDocValuesGen();
final String segmentSuffix = Long.toString(nextDocValuesGen, Character.MAX_RADIX);
final IOContext updatesContext = new IOContext(new FlushInfo(info.info.maxDoc(), bytes));
final IOContext updatesContext = IOContext.flush(new FlushInfo(info.info.maxDoc(), bytes));
final FieldInfo fieldInfo = infos.fieldInfo(field);
assert fieldInfo != null;
fieldInfo.setDocValuesGen(nextDocValuesGen);
Expand Down Expand Up @@ -536,7 +536,7 @@ private synchronized Set<String> writeFieldInfosGen(
// HEADER + FOOTER: 40
// 90 bytes per-field (over estimating long name and attributes map)
final long estInfosSize = 40 + 90L * fieldInfos.size();
final IOContext infosContext = new IOContext(new FlushInfo(info.info.maxDoc(), estInfosSize));
final IOContext infosContext = IOContext.flush(new FlushInfo(info.info.maxDoc(), estInfosSize));
// separately also track which files were created for this gen
final TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(dir);
infosFormat.write(trackingDir, info.info, segmentSuffix, fieldInfos, infosContext);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ void flush(
@Override
void initTermVectorsWriter() throws IOException {
if (writer == null) {
IOContext context = new IOContext(new FlushInfo(lastDocID, bytesUsed.get()));
IOContext context = IOContext.flush(new FlushInfo(lastDocID, bytesUsed.get()));
tmpDirectory = new TrackingTmpOutputDirectoryWrapper(directory);
writer = TEMP_TERM_VECTORS_FORMAT.vectorsWriter(tmpDirectory, info, context);
lastDocID = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ void fill(int docID) throws IOException {

void initTermVectorsWriter() throws IOException {
if (writer == null) {
IOContext context = new IOContext(new FlushInfo(lastDocID, bytesUsed.get()));
IOContext context = IOContext.flush(new FlushInfo(lastDocID, bytesUsed.get()));
writer = codec.termVectorsFormat().vectorsWriter(directory, info, context);
lastDocID = 0;
accountable = writer;
Expand Down
23 changes: 23 additions & 0 deletions lucene/core/src/java/org/apache/lucene/store/DataAccessHint.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;

/** Hint on the data access pattern likely to be used */
public enum DataAccessHint {
RANDOM,
SEQUENTIAL
}
59 changes: 59 additions & 0 deletions lucene/core/src/java/org/apache/lucene/store/DefaultIOContext.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;

import java.util.Arrays;
import java.util.Objects;
import org.apache.lucene.util.Constants;

record DefaultIOContext(ReadAdvice readAdvice, FileOpenHint... hints) implements IOContext {

public DefaultIOContext {
Objects.requireNonNull(readAdvice);
Objects.requireNonNull(hints);

// either hints or readadvice should be specified, not both
if (hints.length > 0 && readAdvice != Constants.DEFAULT_READADVICE) {
throw new IllegalArgumentException("Either readAdvice or hints should be specified");
}
}

@Override
public Context context() {
return Context.DEFAULT;
}

@Override
public MergeInfo mergeInfo() {
return null;
}

@Override
public FlushInfo flushInfo() {
return null;
}

private static final DefaultIOContext[] READADVICE_TO_IOCONTEXT =
Arrays.stream(ReadAdvice.values())
.map(DefaultIOContext::new)
.toArray(DefaultIOContext[]::new);

@Override
public DefaultIOContext withReadAdvice(ReadAdvice advice) {
return READADVICE_TO_IOCONTEXT[advice.ordinal()];
}
}
26 changes: 26 additions & 0 deletions lucene/core/src/java/org/apache/lucene/store/Directory.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.util.Arrays;
import java.util.Collection; // for javadocs
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.util.IOUtils;

Expand Down Expand Up @@ -79,6 +83,28 @@ public abstract class Directory implements Closeable {
*/
public abstract long fileLength(String name) throws IOException;

protected void validateFileOpenHints(IOContext context) {
Map<Class<? extends IOContext.FileOpenHint>, List<IOContext.FileOpenHint>> hintClasses =
Arrays.stream(context.hints())
.collect(Collectors.groupingBy(IOContext.FileOpenHint::getClass));

// there should only be one of FileType, FileData, DataAccess
List<IOContext.FileOpenHint> fileTypes =
hintClasses.getOrDefault(FileTypeHint.class, List.of());
if (fileTypes.size() > 1) {
throw new IllegalArgumentException("Multiple file type hints specified: " + fileTypes);
}
List<IOContext.FileOpenHint> fileData = hintClasses.getOrDefault(FileDataHint.class, List.of());
if (fileData.size() > 1) {
throw new IllegalArgumentException("Multiple file data hints specified: " + fileData);
}
List<IOContext.FileOpenHint> dataAccess =
hintClasses.getOrDefault(DataAccessHint.class, List.of());
if (dataAccess.size() > 1) {
throw new IllegalArgumentException("Multiple data access hints specified: " + dataAccess);
}
}

/**
* Creates a new, empty file in the directory and returns an {@link IndexOutput} instance for
* appending data to this file.
Expand Down
24 changes: 24 additions & 0 deletions lucene/core/src/java/org/apache/lucene/store/FileDataHint.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;

/** Hints on the type of data stored in the file */
public enum FileDataHint implements IOContext.FileOpenHint {
POSTINGS,
STORED_FIELDS,
VECTORS
}
24 changes: 24 additions & 0 deletions lucene/core/src/java/org/apache/lucene/store/FileTypeHint.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.store;

/** Hints on the type of file being opened */
public enum FileTypeHint implements IOContext.FileOpenHint {
METADATA,
DATA,
INDEX
}
Loading