MOD-6786 Fix search on larger then 128 terms (#5524)

lerman25 · web-flow · commit efda03de8780 · 2025-01-22T16:28:35.000Z
* Move length slicing to NOMODIFY if

* add py test

* fix slicing

* fix test

* fix text skip cluster

* Adding comments

* Update test_issues - skip cluster
diff --git a/src/tokenize.c b/src/tokenize.c
@@ -82,12 +82,14 @@ uint32_t simpleTokenizer_Next(RSTokenizer *base, Token *t) {
     char *tok = toksep(&self->pos, &origLen);
     // normalize the token
     size_t normLen = origLen;
-    if (normLen > MAX_NORMALIZE_SIZE) {
-      normLen = MAX_NORMALIZE_SIZE;
-    }
     char normalized_s[MAX_NORMALIZE_SIZE];
     char *normBuf;
-    if (ctx->options & TOKENIZE_NOMODIFY) {
+
+    if (ctx->options & TOKENIZE_NOMODIFY) { // This is a dead code
+      // The stack MAX_NORMALIZE_SIZE buffer is used only if we don't modify the token, for stack allocation safety
+      if (normLen > MAX_NORMALIZE_SIZE) {
+        normLen = MAX_NORMALIZE_SIZE;
+      }
       normBuf = normalized_s;
     } else {
       normBuf = tok;
diff --git a/tests/pytests/test_issues.py b/tests/pytests/test_issues.py
@@ -1313,3 +1313,17 @@ def test_mod_8568(env:Env):
   env.expect('FT.SEARCH', 'idx', '*', 'GEOFILTER', 'g', '1.1', '1.1', '1', 'km').equal(expected)
   env.expect('FT.SEARCH', 'idx', '*', 'GEOFILTER', 'g', '1.1', '1.1', '1', 'km',
                                       'GEOFILTER', 'g', '1.1', '1.1', '1000', 'km').equal(expected)
+ 
+@skip(cluster=True)
+def test_mod_6786(env:Env):
+  # Test search of long term (>128) inside text field
+  MAX_NORMALIZE_SIZE = 128
+  env.expect('FT.CREATE', 'idx', 'SCHEMA', 't', 'TEXT').ok()
+
+  long_term = 'A'*(MAX_NORMALIZE_SIZE+1)
+  text_with_long_term = ' '.join([long_term, long_term[:MAX_NORMALIZE_SIZE//2]])
+  env.cmd('HSET', 'doc1', 't', text_with_long_term)
+
+  # Searching for the long term should return the document
+  # Before fix, the long term was partialy normalized and the document was not found
+  env.expect('FT.SEARCH', 'idx', long_term).equal([1, 'doc1', ['t', text_with_long_term]])
