pip install tokenizers

--tokenizer-pool-size
Size of tokenizer pool to use for asynchronous tokenization. If 0, will use synchronous tokenization.

Default: 0

--tokenizer-pool-type
Type of tokenizer pool to use for asynchronous tokenization. Ignored if tokenizer_pool_size is 0.

Default: “ray”

--tokenizer-pool-extra-config
Extra config for tokenizer pool. This should be a JSON string that will be parsed into a dictionary. Ignored if tokenizer_pool_size is 0.

await loop.run_in_executor(
                    executor, 
                    lambda: self.tokenizer.encode_batch_fast(large_batch)
                )

try:
            executor = concurrent.futures.ThreadPoolExecutor(
                max_workers=2048
            )
            loop = asyncio.get_running_loop()

            async def encode_sync_with_executor(_):
                # Use the pre-initialized executor
                return await loop.run_in_executor(
                    executor, 
                    lambda: self.tokenizer.encode_batch_fast(large_batch)
                )
                
            async def encode_to_thread_sync(_):
                return await asyncio.to_thread(
                    self.tokenizer.encode_batch_fast, large_batch
                )
            
            async def encode_async(_):
                return await self.tokenizer.async_encode_batch_fast(large_batch)
            
            await asyncio.gather(*[encode_sync_with_executor(i) for i in range(2048)])
            await asyncio.gather(*[encode_async(i) for i in range(2048)])
            
            for n_tasks in [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]:
                # Measure sync performance with pre-initialized executor
                # Warm up
                await asyncio.gather(*[encode_sync_with_executor(i) for i in range(10)])
                time.sleep(0.03)
                # Actual measurement
                start = time.perf_counter()
                await asyncio.gather(*[encode_sync_with_executor(i) for i in range(n_tasks)])
                sync_time = time.perf_counter() - start

                # Measure async performance
                # Warm up
                await asyncio.gather(*[encode_async(i) for i in range(10)])
                
                # Actual measurement
                time.sleep(0.03)
                start = time.perf_counter()
                await asyncio.gather(*[encode_async(i) for i in range(n_tasks)])
                async_time = time.perf_counter() - start
                
                # Log times
                print(f"sync vs async processing times: {sync_time:.4f}s vs {async_time:.4f}s for {n_tasks} tasks")
                results_sync.append(sync_time)
                results_async.append(async_time)
        finally:
            # Make sure we shut down the executor properly
            executor.shutdown(wait=False)

sync vs async processing times: 0.0003s vs 0.0004s for 1 tasks
sync vs async processing times: 0.0004s vs 0.0005s for 2 tasks
sync vs async processing times: 0.0004s vs 0.0007s for 4 tasks
sync vs async processing times: 0.0014s vs 0.0011s for 8 tasks
sync vs async processing times: 0.0028s vs 0.0022s for 16 tasks
sync vs async processing times: 0.0057s vs 0.0040s for 32 tasks
sync vs async processing times: 0.0097s vs 0.0065s for 64 tasks
sync vs async processing times: 0.0172s vs 0.0127s for 128 tasks
sync vs async processing times: 0.0324s vs 0.0293s for 256 tasks
sync vs async processing times: 0.0528s vs 0.0487s for 512 tasks
sync vs async processing times: 0.1549s vs 0.0960s for 1024 tasks
sync vs async processing times: 0.3044s vs 0.1846s for 2048 tasks