maph 1.0.0 High-Performance Memory-Mapped Key-Value Database with O(1) Lookups

Loading...
Searching...
No Matches
maph_cli.cpp
Go to the documentation of this file.
1
18#include "maph.hpp"
19#include <iostream>
20#include <fstream>
21#include <chrono>
22#include <cstring>
23#include <iomanip> // For std::setprecision
24#include <thread> // For std::thread::hardware_concurrency
25
26 using namespace maph;
27
28// Exit codes for consistent error handling
29 constexpr int EXIT_SUCCESS_CODE = 0;
30 constexpr int EXIT_ERROR_CODE = 1;
31 constexpr int EXIT_INVALID_ARGS = 2;
32 constexpr int EXIT_FILE_ERROR = 3;
33 constexpr int EXIT_DATABASE_FULL = 4;
34
41 void usage() {
42 std::cerr << R"(maph - Memory-mapped Approximate Perfect Hash
43
44 COMMANDS:
45 create <file> <slots> Create new maph file
46 set <file> <key> <value> Set key-value pair
47 get <file> <key> Get value for key
48 remove <file> <key> Remove key
49 stats <file> Show statistics
50 bench <file> Run benchmark
51 bench_parallel <file> [threads] Run parallel benchmark
52 load_bulk <file> <jsonl> Load JSONL file in parallel
53 mget <file> <key1> ... Get multiple keys
54 mset <file> k1 v1 k2 v2... Set multiple key-value pairs
55
56 OPTIONS:
57 --threads <n> Thread count for parallel ops
58 --durability <ms> Enable async durability
59
60 EXAMPLES:
61 maph create data.maph 1000000
62 maph set data.maph '{"id":1}' '{"name":"alice"}'
63 maph get data.maph '{"id":1}'
64 maph bench_parallel data.maph 8
65 maph load_bulk data.maph input.jsonl --threads 4
66)";
67}
68
80 int main(int argc, char* argv[]) {
81 if (argc < 2) {
82 usage();
83 return EXIT_INVALID_ARGS;
84 }
85
86 std::string cmd = argv[1];
87
88 // Handle help request
89 if (cmd == "--help" || cmd == "-h") {
90 usage();
91 return EXIT_SUCCESS_CODE;
92 }
93
102 if (cmd == "create" && argc == 4) {
103 try {
104 uint64_t num_slots = std::stoull(argv[3]);
105
106 // Validate slot count
107 if (num_slots == 0) {
108 std::cerr << "Error: Number of slots must be greater than 0\n";
109 return EXIT_INVALID_ARGS;
110 }
111
112 auto m = Maph::create(argv[2], num_slots);
113 if (!m) {
114 std::cerr << "Failed to create " << argv[2] << "\n";
115 std::cerr << "Check disk space and permissions\n";
116 return EXIT_FILE_ERROR;
117 }
118
119 // Calculate and display file size
120 size_t file_size = sizeof(Header) + (num_slots * sizeof(Slot));
121 std::cout << "Created " << argv[2] << " with " << num_slots << " slots\n";
122 std::cout << "File size: " << (file_size / (1024*1024)) << " MB\n";
123 return EXIT_SUCCESS_CODE;
124 } catch (const std::exception& e) {
125 std::cerr << "Error: " << e.what() << "\n";
126 return EXIT_ERROR_CODE;
127 }
128 }
129
138 if (cmd == "set" && argc == 5) {
139 auto m = open(argv[2]);
140 if (!m) {
141 std::cerr << "Failed to open " << argv[2] << "\n";
142 std::cerr << "Check if file exists and has correct permissions\n";
143 return EXIT_FILE_ERROR;
144 }
145
146 // Check value size
147 std::string_view value(argv[4]);
148 if (value.size() > Slot::MAX_SIZE) {
149 std::cerr << "Error: Value too large (" << value.size()
150 << " bytes, max " << Slot::MAX_SIZE << ")\n";
151 return EXIT_INVALID_ARGS;
152 }
153
154 if (m->set(argv[3], argv[4])) {
155 std::cout << "OK\n";
156 return EXIT_SUCCESS_CODE;
157 } else {
158 std::cerr << "Failed to set - database may be full\n";
159 return EXIT_DATABASE_FULL;
160 }
161 }
162
172 if (cmd == "get" && argc == 4) {
173 auto m = open_readonly(argv[2]);
174 if (!m) {
175 std::cerr << "Failed to open " << argv[2] << "\n";
176 return EXIT_FILE_ERROR;
177 }
178
179 if (auto value = m->get(argv[3])) {
180 // Value found - print to stdout
181 std::cout << *value << "\n";
182 return EXIT_SUCCESS_CODE;
183 } else {
184 // Key not found - print null and return error
185 std::cout << "null\n";
186 return EXIT_ERROR_CODE;
187 }
188 }
189
198 if (cmd == "remove" && argc == 4) {
199 auto m = open(argv[2]);
200 if (!m) {
201 std::cerr << "Failed to open " << argv[2] << "\n";
202 return EXIT_FILE_ERROR;
203 }
204
205 if (m->remove(argv[3])) {
206 std::cout << "OK\n";
207 return EXIT_SUCCESS_CODE;
208 } else {
209 std::cerr << "Not found\n";
210 return EXIT_ERROR_CODE;
211 }
212 }
213
222 if (cmd == "stats" && argc == 3) {
223 auto m = open_readonly(argv[2]);
224 if (!m) {
225 std::cerr << "Failed to open " << argv[2] << "\n";
226 return EXIT_FILE_ERROR;
227 }
228
229 auto s = m->stats();
230
231 // Display statistics in readable format
232 std::cout << "Database: " << argv[2] << "\n";
233 std::cout << "======================\n";
234 std::cout << "Total slots: " << s.total_slots << "\n";
235 std::cout << "Static slots: " << s.static_slots
236 << " (" << (s.static_slots * 100.0 / s.total_slots) << "%)\n";
237 std::cout << "Used slots: " << s.used_slots
238 << " (" << (s.used_slots * 100.0 / s.total_slots) << "%)\n";
239 std::cout << "Free slots: " << (s.total_slots - s.used_slots) << "\n";
240 std::cout << "Load factor: " << std::fixed << std::setprecision(4) << s.load_factor << "\n";
241 std::cout << "Memory: " << s.memory_bytes / (1024*1024) << " MB\n";
242 std::cout << "Generation: " << s.generation << "\n";
243
244 // Warn if database is getting full
245 if (s.load_factor > 0.8) {
246 std::cerr << "\nWARNING: Database is "
247 << (s.load_factor * 100) << "% full\n";
248 }
249
250 return EXIT_SUCCESS_CODE;
251 }
252
262 if (cmd == "bench" && argc == 3) {
263 auto m = open(argv[2]);
264 if (!m) {
265 std::cerr << "Failed to open " << argv[2] << "\n";
266 return EXIT_FILE_ERROR;
267 }
268
269 const int N = 100000; // Number of operations for benchmark
270
271 std::cout << "Running benchmark with " << N << " operations...\n";
272 auto start = std::chrono::high_resolution_clock::now();
273
274 // Write benchmark - test sequential insertion performance
275 for (int i = 0; i < N; ++i) {
276 std::string key = "{\"id\":" + std::to_string(i) + "}";
277 std::string val = "{\"v\":" + std::to_string(i*10) + "}";
278 if (!m->set(key, val)) {
279 std::cerr << "Warning: Write failed at iteration " << i << "\n";
280 }
281 }
282
283 auto mid = std::chrono::high_resolution_clock::now();
284
285 // Read benchmark - test sequential retrieval performance
286 // Use volatile to prevent compiler optimization
287 int found = 0;
288 for (int i = 0; i < N; ++i) {
289 std::string key = "{\"id\":" + std::to_string(i) + "}";
290 volatile auto v = m->get(key);
291 if (v) found++;
292 }
293
294 auto end = std::chrono::high_resolution_clock::now();
295
296 // Calculate performance metrics
297 auto write_us = std::chrono::duration_cast<std::chrono::microseconds>(mid - start).count();
298 auto read_us = std::chrono::duration_cast<std::chrono::microseconds>(end - mid).count();
299
300 // Display results
301 std::cout << "\nBenchmark Results:\n";
302 std::cout << "==================\n";
303 std::cout << "Write Performance:\n";
304 std::cout << " Operations: " << N << "\n";
305 std::cout << " Time: " << write_us / 1000.0 << " ms\n";
306 std::cout << " Throughput: " << std::fixed << std::setprecision(0)
307 << (N * 1000000.0 / write_us) << " ops/sec\n";
308 std::cout << " Latency: " << (write_us * 1000 / N) << " ns/op\n\n";
309
310 std::cout << "Read Performance:\n";
311 std::cout << " Operations: " << N << "\n";
312 std::cout << " Found: " << found << "/" << N << "\n";
313 std::cout << " Time: " << read_us / 1000.0 << " ms\n";
314 std::cout << " Throughput: " << std::fixed << std::setprecision(0)
315 << (N * 1000000.0 / read_us) << " ops/sec\n";
316 std::cout << " Latency: " << (read_us * 1000 / N) << " ns/op\n";
317
318 return EXIT_SUCCESS_CODE;
319 }
320
330 if (cmd == "bench_parallel" && (argc == 3 || argc == 4)) {
331 auto m = open(argv[2]);
332 if (!m) {
333 std::cerr << "Failed to open " << argv[2] << "\n";
334 return EXIT_FILE_ERROR;
335 }
336
337 // Parse thread count or use hardware default
338 size_t threads = (argc == 4) ? std::stoull(argv[3]) : std::thread::hardware_concurrency();
339 const int N = 100000; // Total operations to perform
340
341 std::cout << "Running parallel benchmark with " << threads << " threads...\n";
342 std::cout << "Operations per thread: " << (N / threads) << "\n\n";
343
344 // Prepare batch data - pre-allocate to avoid allocation during benchmark
345 std::vector<std::pair<JsonView, JsonView>> kvs;
346 std::vector<std::string> keys, values;
347 keys.reserve(N);
348 values.reserve(N);
349
350 for (int i = 0; i < N; ++i) {
351 keys.push_back("{\"id\":" + std::to_string(i) + "}");
352 values.push_back("{\"v\":" + std::to_string(i*10) + "}");
353 }
354
355 // Create views for zero-copy operations
356 for (int i = 0; i < N; ++i) {
357 kvs.emplace_back(keys[i], values[i]);
358 }
359
360 // Parallel write benchmark - distribute writes across threads
361 auto start = std::chrono::high_resolution_clock::now();
362 size_t written = m->parallel_mset(kvs, threads);
363 auto mid = std::chrono::high_resolution_clock::now();
364
365 // Parallel read benchmark - distribute reads across threads
366 std::vector<JsonView> key_views;
367 key_views.reserve(keys.size());
368 for (const auto& k : keys) {
369 key_views.push_back(k);
370 }
371
372 // Count successful reads using atomic counter
373 std::atomic<size_t> count{0};
374 m->parallel_mget(key_views,
375 [&count](JsonView k, JsonView v) {
376 count.fetch_add(1, std::memory_order_relaxed);
377 },
378 threads);
379 auto end = std::chrono::high_resolution_clock::now();
380
381 // Calculate and display performance metrics
382 auto write_us = std::chrono::duration_cast<std::chrono::microseconds>(mid - start).count();
383 auto read_us = std::chrono::duration_cast<std::chrono::microseconds>(end - mid).count();
384
385 std::cout << "Parallel Benchmark Results (" << threads << " threads):\n";
386 std::cout << "======================================\n";
387 std::cout << "Write Performance:\n";
388 std::cout << " Operations: " << N << "\n";
389 std::cout << " Successful: " << written << "\n";
390 std::cout << " Time: " << write_us / 1000.0 << " ms\n";
391 std::cout << " Throughput: " << std::fixed << std::setprecision(0)
392 << (N * 1000000.0 / write_us) << " ops/sec\n";
393 std::cout << " Per-thread: " << std::fixed << std::setprecision(0)
394 << (N * 1000000.0 / write_us / threads) << " ops/sec/thread\n\n";
395
396 std::cout << "Read Performance:\n";
397 std::cout << " Operations: " << N << "\n";
398 std::cout << " Found: " << count.load() << "\n";
399 std::cout << " Time: " << read_us / 1000.0 << " ms\n";
400 std::cout << " Throughput: " << std::fixed << std::setprecision(0)
401 << (N * 1000000.0 / read_us) << " ops/sec\n";
402 std::cout << " Per-thread: " << std::fixed << std::setprecision(0)
403 << (N * 1000000.0 / read_us / threads) << " ops/sec/thread\n";
404
405 // Calculate speedup
406 std::cout << "\nSpeedup vs single thread: "
407 << std::fixed << std::setprecision(2)
408 << threads << "x theoretical, "
409 << (N * 1000000.0 / read_us) / (5000000.0) << "x actual\n";
410
411 return EXIT_SUCCESS_CODE;
412 }
413
423 if (cmd == "load_bulk" && argc >= 4) {
424 auto m = open(argv[2]);
425 if (!m) {
426 std::cerr << "Failed to open database " << argv[2] << "\n";
427 return EXIT_FILE_ERROR;
428 }
429
430 size_t threads = std::thread::hardware_concurrency();
431 // Check for --threads option
432 for (int i = 4; i < argc - 1; ++i) {
433 if (std::string(argv[i]) == "--threads") {
434 threads = std::stoull(argv[i + 1]);
435 break;
436 }
437 }
438
439 std::ifstream file(argv[3]);
440 if (!file) {
441 std::cerr << "Failed to open " << argv[3] << "\n";
442 return 1;
443 }
444
445 // Load all lines first
446 std::vector<std::pair<std::string, std::string>> kvs;
447 std::string line;
448 while (std::getline(file, line)) {
449 // Parse JSONL: expect {"input": key, "output": value}
450 size_t input_pos = line.find("\"input\":");
451 size_t output_pos = line.find("\"output\":");
452 if (input_pos != std::string::npos && output_pos != std::string::npos) {
453 // Simple extraction (assumes no nested quotes)
454 size_t key_start = input_pos + 8;
455 size_t key_end = line.find(',', key_start);
456 size_t val_start = output_pos + 9;
457 size_t val_end = line.find('}', val_start);
458
459 std::string key = line.substr(key_start, key_end - key_start);
460 std::string val = line.substr(val_start, val_end - val_start);
461
462 // Trim quotes if present
463 if (key.front() == '"') key = key.substr(1, key.size() - 2);
464 if (val.front() == '"') val = val.substr(1, val.size() - 2);
465
466 kvs.emplace_back(key, val);
467 }
468 }
469
470 std::cout << "Loading " << kvs.size() << " entries with " << threads << " threads...\n";
471
472 auto start = std::chrono::high_resolution_clock::now();
473
474 // Convert to views for parallel_mset
475 std::vector<std::pair<JsonView, JsonView>> kv_views;
476 for (const auto& [k, v] : kvs) {
477 kv_views.emplace_back(k, v);
478 }
479
480 size_t loaded = m->parallel_mset(kv_views, threads);
481
482 auto end = std::chrono::high_resolution_clock::now();
483 auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
484
485 std::cout << "Loaded " << loaded << "/" << kvs.size() << " entries in " << ms << " ms\n";
486 std::cout << "Rate: " << (loaded * 1000.0 / ms) << " entries/sec\n";
487 return 0;
488 }
489
499 if (cmd == "mget" && argc >= 4) {
500 auto m = open_readonly(argv[2]);
501 if (!m) {
502 std::cerr << "Failed to open " << argv[2] << "\n";
503 return EXIT_FILE_ERROR;
504 }
505
506 // Collect all keys from command line
507 std::vector<JsonView> keys;
508 keys.reserve(argc - 3);
509 for (int i = 3; i < argc; ++i) {
510 keys.emplace_back(argv[i]);
511 }
512
513 // Perform batch retrieval with callback for each found pair
514 size_t found = 0;
515 m->mget(keys, [&found](JsonView key, JsonView value) {
516 std::cout << key << ": " << value << "\n";
517 found++;
518 });
519
520 // Report summary
521 if (found < keys.size()) {
522 std::cerr << "\nFound " << found << "/" << keys.size() << " keys\n";
523 }
524
525 return (found == keys.size()) ? EXIT_SUCCESS_CODE : EXIT_ERROR_CODE;
526 }
527
537 if (cmd == "mset" && argc >= 5 && (argc - 3) % 2 == 0) {
538 auto m = open(argv[2]);
539 if (!m) {
540 std::cerr << "Failed to open " << argv[2] << "\n";
541 return EXIT_FILE_ERROR;
542 }
543
544 // Collect key-value pairs from command line
545 std::vector<std::pair<JsonView, JsonView>> kvs;
546 kvs.reserve((argc - 3) / 2);
547
548 // Validate value sizes before attempting batch operation
549 for (int i = 3; i < argc; i += 2) {
550 std::string_view value(argv[i + 1]);
551 if (value.size() > Slot::MAX_SIZE) {
552 std::cerr << "Error: Value for key '" << argv[i]
553 << "' too large (" << value.size() << " bytes)\n";
554 return EXIT_INVALID_ARGS;
555 }
556 kvs.emplace_back(argv[i], argv[i + 1]);
557 }
558
559 // Perform batch insertion
560 size_t count = m->mset(kvs);
561 std::cout << "Stored " << count << "/" << kvs.size() << " pairs\n";
562
563 if (count < kvs.size()) {
564 std::cerr << "Warning: " << (kvs.size() - count)
565 << " pairs failed (database may be full)\n";
566 return EXIT_DATABASE_FULL;
567 }
568
569 return EXIT_SUCCESS_CODE;
570 }
571
572 // If no command matched, show usage
573 std::cerr << "Error: Unknown command '" << cmd << "'\n\n";
574 usage();
575 return EXIT_INVALID_ARGS;
576}
maph::Maph::create
static std::unique_ptr< Maph > create(const std::string &path, uint64_t total_slots, uint64_t static_slots=0)
Create a new database file.
Definition maph.hpp:387
main
int main()
Definition echo_server.c:122
maph.hpp
Lean, mean, mmap-based perfect hash JSON mapping.
EXIT_FILE_ERROR
constexpr int EXIT_FILE_ERROR
Definition maph_cli.cpp:32
usage
void usage()
Display comprehensive usage information.
Definition maph_cli.cpp:41
EXIT_DATABASE_FULL
constexpr int EXIT_DATABASE_FULL
Definition maph_cli.cpp:33
EXIT_SUCCESS_CODE
constexpr int EXIT_SUCCESS_CODE
Definition maph_cli.cpp:29
EXIT_ERROR_CODE
constexpr int EXIT_ERROR_CODE
Definition maph_cli.cpp:30
EXIT_INVALID_ARGS
constexpr int EXIT_INVALID_ARGS
Definition maph_cli.cpp:31
maph
Definition maph.hpp:46
maph::open
auto open(const std::string &path)
Definition maph.hpp:958
maph::open_readonly
auto open_readonly(const std::string &path)
Definition maph.hpp:962
maph::JsonView
std::string_view JsonView
Definition maph.hpp:48
maph::Header
Database file header containing metadata.
Definition maph.hpp:196
maph::Slot
Storage slot for key-value pairs.
Definition maph.hpp:115
maph::Slot::MAX_SIZE
static constexpr size_t MAX_SIZE
Maximum value size (496 bytes)
Definition maph.hpp:121