Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
0e97c61
refactor intersection iterator
GuyAv46 Feb 4, 2025
ea02df9
WIP reimplement union iterator
GuyAv46 Feb 4, 2025
f2c2b99
wip reimplement not iterator
GuyAv46 Feb 4, 2025
74f45a5
WIP refactor structs and API
GuyAv46 Feb 6, 2025
d018e7b
fix tests
GuyAv46 Feb 6, 2025
e39be8c
cleanups
GuyAv46 Feb 6, 2025
3204ec6
WIP union cleanup
GuyAv46 Feb 6, 2025
cce47e6
minor tidy up
GuyAv46 Feb 6, 2025
c092e1c
minor tidy ups
GuyAv46 Feb 6, 2025
0d43532
support timeout return code
GuyAv46 Feb 7, 2025
9f3cb7f
fix ir read and skip-to contract
GuyAv46 Feb 7, 2025
daf2aa2
fixes
GuyAv46 Feb 7, 2025
2f642bb
reimplemented not iterator (opt and reg)
GuyAv46 Feb 7, 2025
74aff73
some fixes
GuyAv46 Feb 7, 2025
21fa006
fix not skipto
GuyAv46 Feb 7, 2025
21893d6
fix list iterator skip-to
GuyAv46 Feb 7, 2025
157a485
some more fixes
GuyAv46 Feb 7, 2025
28c7df4
remove has-next API
GuyAv46 Feb 8, 2025
0f1ca98
some test fixes
GuyAv46 Feb 8, 2025
2fd1882
union full refactor
GuyAv46 Feb 8, 2025
d7af99e
aligned optional implementation
GuyAv46 Feb 8, 2025
8730c73
fix name
GuyAv46 Feb 8, 2025
2cbe60f
improve comment
GuyAv46 Feb 8, 2025
19670e8
fixes
GuyAv46 Feb 8, 2025
4b19cd9
fix unit tests
GuyAv46 Feb 8, 2025
1eaa24c
fix optimizer eof
GuyAv46 Feb 8, 2025
8ab45a1
fix test
GuyAv46 Feb 8, 2025
56a763b
remove redundant check
GuyAv46 Feb 9, 2025
ef31bf4
remove and improve abort API
GuyAv46 Feb 9, 2025
d30f93e
improve abort removal
GuyAv46 Feb 9, 2025
2320f7f
minor cleanup
GuyAv46 Feb 9, 2025
42f9b10
reorder and optimize intersection iterator
GuyAv46 Feb 10, 2025
78193a0
improve union performance
GuyAv46 Feb 11, 2025
9abd847
simplified
GuyAv46 Feb 11, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/debug_commands.c
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ static void ReplyReaderResults(IndexReader *reader, RedisModuleCtx *ctx) {
RSIndexResult *r;
size_t resultSize = 0;
RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN);
while (iter->Read(iter->ctx, &r) != INDEXREAD_EOF) {
while (iter->Read(iter, &r) != INDEXREAD_EOF) {
RedisModule_ReplyWithLongLong(ctx, r->docId);
++resultSize;
}
Expand Down Expand Up @@ -324,12 +324,12 @@ InvertedIndexStats InvertedIndex_DebugReply(RedisModuleCtx *ctx, InvertedIndex *
REPLY_WITH_STR("values", ARRAY_LEN_VAR(invertedIndexDump));
START_POSTPONED_LEN_ARRAY(invertedIndexValues);
RSIndexResult *res = NULL;
IndexReader *ir = NewMinimalNumericReader(idx, false);
while (INDEXREAD_OK == IR_Read(ir, &res)) {
IndexIterator *ir = NewReadIterator(NewMinimalNumericReader(idx, false));
while (INDEXREAD_OK == ir->Read(ir, &res)) {
REPLY_WITH_DOUBLE("value", res->num.value, ARRAY_LEN_VAR(invertedIndexValues));
REPLY_WITH_LONG_LONG("docId", res->docId, ARRAY_LEN_VAR(invertedIndexValues));
}
IR_Free(ir);
ir->Free(ir);
END_POSTPONED_LEN_ARRAY(invertedIndexValues);
ARRAY_LEN_VAR(invertedIndexDump)++;

Expand Down
4 changes: 2 additions & 2 deletions src/fork_gc.c
Original file line number Diff line number Diff line change
Expand Up @@ -791,15 +791,15 @@ static void resetCardinality(NumGcInfo *info, NumericRange *range, size_t blocks
}
// Add the entries that were added since the fork to the HLL
RSIndexResult *cur;
IndexReader *ir = NewMinimalNumericReader(range->entries, false);
IndexIterator *ir = NewReadIterator(NewMinimalNumericReader(range->entries, false));
size_t startIdx = range->entries->size - blocksSinceFork; // Here `blocksSinceFork` > 0
t_docId startId = range->entries->blocks[startIdx].firstId;
int rc = IR_SkipTo(ir, startId, &cur);
while (INDEXREAD_OK == rc) {
hll_add(&range->hll, &cur->num.value, sizeof(cur->num.value));
rc = IR_Read(ir, &cur);
}
IR_Free(ir);
ReadIterator_Free(ir);
}

static void applyNumIdx(ForkGC *gc, RedisSearchCtx *sctx, NumGcInfo *ninfo) {
Expand Down
2 changes: 1 addition & 1 deletion src/geo_index.c
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ IndexIterator *NewGeoRangeIterator(const RedisSearchCtx *ctx, const GeoFilter *g
NewNumericFilter(ranges[ii].min, ranges[ii].max, 1, 1, true);
filt->field = gf->field;
filt->geoFilter = gf;
struct indexIterator *numIter = NewNumericFilterIterator(ctx, filt, csx, INDEXFLD_T_GEO, config, &filterCtx);
IndexIterator *numIter = NewNumericFilterIterator(ctx, filt, csx, INDEXFLD_T_GEO, config, &filterCtx);
if (numIter != NULL) {
iters[itersCount++] = numIter;
}
Expand Down
57 changes: 22 additions & 35 deletions src/geometry/query_iterator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
}

int QueryIterator::read_single(RSIndexResult *&hit) noexcept {
if (!base_.isValid || !has_next()) {
if (!base_.isValid) {
return INDEXREAD_EOF;
}
t_docId docId = iter_[index_++];
Expand All @@ -35,6 +35,10 @@
}

int QueryIterator::read(RSIndexResult *&hit) noexcept {
if (index_ >= len()) {
base_.isValid = false;
return INDEXREAD_EOF;
}
size_t timeoutCounter = 0;
int rc = INDEXREAD_OK;
do {
Expand All @@ -46,7 +50,7 @@
return rc;
}
int QueryIterator::skip_to(t_docId docId, RSIndexResult *&hit) {
if (!base_.isValid || !has_next()) {
if (!base_.isValid) {
return INDEXREAD_EOF;
}
if (docId > iter_.back()) {
Expand All @@ -57,8 +61,8 @@
const auto it = std::ranges::lower_bound(std::ranges::next(std::ranges::begin(iter_), index_),
std::ranges::end(iter_), docId);
index_ = std::ranges::distance(std::ranges::begin(iter_), it + 1);
if (!has_next()) {
abort();
if (index_ >= len()) {
base_.isValid = false;
}

base_.current->docId = *it;
Expand All @@ -72,68 +76,51 @@
t_docId QueryIterator::current() const noexcept {
return base_.current->docId;
}
int QueryIterator::has_next() const noexcept {
return index_ < len();
}
std::size_t QueryIterator::len() const noexcept {
return iter_.size();
}
void QueryIterator::abort() noexcept {
base_.isValid = false;
}
void QueryIterator::rewind() noexcept {
base_.isValid = true;
base_.current->docId = 0;
base_.LastDocId = 0;

Check warning on line 85 in src/geometry/query_iterator.cpp

View check run for this annotation

Codecov / codecov/patch

src/geometry/query_iterator.cpp#L85

Added line #L85 was not covered by tests
index_ = 0;
}

namespace {
int QIter_Read(void *ctx, RSIndexResult **hit) {
return static_cast<QueryIterator *>(ctx)->read(*hit);
int QIter_Read(IndexIterator *base, RSIndexResult **hit) {
return reinterpret_cast<QueryIterator *>(base)->read(*hit);
}
int QIter_SkipTo(void *ctx, t_docId docId, RSIndexResult **hit) {
return static_cast<QueryIterator *>(ctx)->skip_to(docId, *hit);
}
t_docId QIter_LastDocId(void *ctx) {
return static_cast<QueryIterator const *>(ctx)->current();
}
int QIter_HasNext(void *ctx) {
return static_cast<QueryIterator const *>(ctx)->has_next();
int QIter_SkipTo(IndexIterator *base, t_docId docId, RSIndexResult **hit) {
return reinterpret_cast<QueryIterator *>(base)->skip_to(docId, *hit);
}
void QIter_Free(IndexIterator *self) {
using alloc_type = Allocator::TrackingAllocator<QueryIterator>;
const auto qi = static_cast<QueryIterator *const>(self->ctx);
const auto qi = reinterpret_cast<QueryIterator *const>(self);
auto alloc = alloc_type{qi->iter_.get_allocator()};
IndexResult_Free(self->current);
std::allocator_traits<alloc_type>::destroy(alloc, qi);
std::allocator_traits<alloc_type>::deallocate(alloc, qi, 1);
}
std::size_t QIter_Len(void *ctx) {
return static_cast<QueryIterator const *>(ctx)->len();
}
void QIter_Abort(void *ctx) {
static_cast<QueryIterator *>(ctx)->abort();
std::size_t QIter_Len(IndexIterator *base) {
return reinterpret_cast<QueryIterator const *>(base)->len();
}
void QIter_Rewind(void *ctx) {
static_cast<QueryIterator *>(ctx)->rewind();
void QIter_Rewind(IndexIterator *base) {
reinterpret_cast<QueryIterator *>(base)->rewind();

Check warning on line 108 in src/geometry/query_iterator.cpp

View check run for this annotation

Codecov / codecov/patch

src/geometry/query_iterator.cpp#L107-L108

Added lines #L107 - L108 were not covered by tests
}

} // anonymous namespace

IndexIterator QueryIterator::init_base(QueryIterator *ctx) {
return IndexIterator{
.isValid = 1,
.ctx = ctx,
.current = NewVirtualResult(0, RS_FIELDMASK_ALL),
.type = ID_LIST_ITERATOR,
.isValid = true,
.isAborted = false,
.LastDocId = 0,
.current = NewVirtualResult(0, RS_FIELDMASK_ALL),
.NumEstimated = QIter_Len,
.Read = QIter_Read,
.SkipTo = QIter_SkipTo,
.LastDocId = QIter_LastDocId,
.HasNext = QIter_HasNext,
.Free = QIter_Free,
.Len = QIter_Len,
.Abort = QIter_Abort,
.Rewind = QIter_Rewind,
};
}
Expand Down
1 change: 0 additions & 1 deletion src/geometry/query_iterator.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ struct QueryIterator {
int read(RSIndexResult *&hit) noexcept;
int skip_to(t_docId docId, RSIndexResult *&hit);
t_docId current() const noexcept;
int has_next() const noexcept;
std::size_t len() const noexcept;
void abort() noexcept;
void rewind() noexcept;
Expand Down
4 changes: 2 additions & 2 deletions src/highlight_processor.c
Original file line number Diff line number Diff line change
Expand Up @@ -277,8 +277,8 @@ static const RSIndexResult *getIndexResult(ResultProcessor *rp, t_docId docId) {
if (!it) {
return NULL;
}
it->Rewind(it->ctx);
if (INDEXREAD_OK != it->SkipTo(it->ctx, docId, &ir)) {
it->Rewind(it);
if (INDEXREAD_OK != it->SkipTo(it, docId, &ir)) {
return NULL;
}
return ir;
Expand Down
Loading