Relax FILE_ERROR handling. Benchmark example

examples/benchmark/bench_data.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python2
+
+'''
+    Based on - https://www.unixuser.org/~euske/doc/cdbinternals/pycdb.py.html
+'''
+
+# calc hash value with a given key
+def calc_hash(s):
+    return reduce(lambda h, c: (((h << 5) + h) ^ ord(c)) & 0xffffffffL, s, 5381)
+
+# cdbmake(filename, hash)
+def cdbmake(f, a):
+    from struct import pack
+
+    # write cdb
+    def write_cdb(fp):
+        pos_header = fp.tell()
+
+        # skip header
+        p = pos_header + 8 * 256
+        fp.seek(p)
+
+        bucket = [[] for i in range(256)]
+
+        # write data & make hash
+        for (k, v) in a.iteritems():
+          fp.write(pack('<LL', len(k), len(v)))
+          fp.write(k)
+          fp.write(v)
+          h = calc_hash(k)
+          bucket[h % 256].append((h, p))
+          p += len(k) + len(v) + 8
+
+        pos_hash = p
+
+        # write hash tables
+        for bt in bucket:
+            if bt:
+                nslots = 2 * len(bt)
+                hash_tab = [(0, 0) for i in range(nslots)]
+                for (h, p) in bt:
+                    i = (h >> 8) % nslots
+                    while hash_tab[i][1]:  # is slot already occupied?
+                        i = (i + 1) % nslots
+                    hash_tab[i] = (h, p)
+                for (h, p) in hash_tab:
+                    fp.write(pack('<LL', h, p))
+
+        # write header
+        fp.seek(pos_header)
+        for bt in bucket:
+            fp.write(pack('<LL', pos_hash, 2 * len(bt)))
+            pos_hash += 16 * len(bt)
+        return
+
+    fp = file(f, "wb")
+    write_cdb(fp)
+    fp.close()
+    return
+
+if __name__ == "__main__":
+
+    bench = {}
+    for i in xrange(0, 5*1000000, 5):
+        bench[str(i)] = str(i)
+    print bench['255']
+    cdbmake("bench.cdb", bench)

examples/benchmark/benchmark.ino

@@ -0,0 +1,145 @@
+/*
+  uCDB benchmark sketch
+  
+  Write file bench.cdb ([0..5000000] divisible by 5) on SD card. Connect to Ardino board.
+  Compile and upload sketch. Open Serial Monitor.
+  
+  Board: Arduino Uno
+  SD card: SDHC 7.31Gb FAT32, sectorsPerCluster - 64
+  SD chip select pin: 10
+  Arduino IDE Serial Monitors settings: 9600 baud, no line ending.
+
+  Created by Ioulianos Kakoulidis, 2021.
+  Released into the public domain.     
+*/
+
+#include "uCDB.h"
+
+void setup() {
+  Serial.begin(9600);
+  while (!Serial) {
+    ;
+  }
+
+  if (!SD.begin(10)) {
+    Serial.println("SD card initialization failed!");
+    while (true) {
+      ;
+    }
+  }
+}
+
+void loop() {
+  uCDB ucdb;  
+  char str[16];
+  long key;
+  unsigned long startMillis;  
+  cdbResult rt;
+
+  Serial.println("Press any key to start test");
+  while (!Serial.available()) {
+    ;
+  }
+
+  if (ucdb.open("bench.cdb") != CDB_OK) {
+    Serial.print("Invalid CDB: ");
+    Serial.println("bench.cdb");
+    return;
+  }
+
+  Serial.println("Querying 1000 random keys from interval [0..5000000)...");
+  startMillis = millis();
+  for (int i = 0; i < 1000; ++i) {
+    key = random(5000000);
+    sprintf(str, "%ld", key);
+    rt = ucdb.findKey(str, strlen(str));
+    if (key%5) {
+      if (rt != KEY_NOT_FOUND) {
+        Serial.print("Error: ");
+        Serial.println(key);
+        break;
+      }
+    }
+    else {
+      if (rt != KEY_FOUND) {
+        Serial.print("Error: ");
+        Serial.println(key);
+        break;
+      }
+    }
+  }
+  Serial.print("Query millis: ");
+  Serial.println(millis() - startMillis);
+
+  Serial.println("Querying 1000 random keys from interval [-5000000,0)...");
+  startMillis = millis();
+  for (int i = 0; i < 1000; ++i) {
+    key = random(-5000000, 0);
+    sprintf(str, "%ld", key);
+    rt = ucdb.findKey(str, strlen(str));
+    if (rt != KEY_NOT_FOUND) {
+      Serial.print("Error: ");
+      Serial.println(str);
+      break;
+    }
+  }
+  Serial.print("Query millis: ");
+  Serial.println(millis() - startMillis);
+
+  Serial.println("Querying 1000 random keys with findNextValue() from interval [0..5000000)...");
+  startMillis = millis();
+  for (int i = 0; i < 1000; ++i) {
+    key = random(5000000);
+    sprintf(str, "%ld", key);
+    rt = ucdb.findKey(str, strlen(str));
+    if (key%5) {
+      if (rt != KEY_NOT_FOUND) {
+        Serial.print("Error: ");
+        Serial.println(key);
+        break;
+      }
+    }
+    else {
+      if (rt != KEY_FOUND) {
+        Serial.print("Error: ");
+        Serial.println(key);
+        break;
+      }
+    }
+    rt = ucdb.findNextValue();
+    if (rt != KEY_NOT_FOUND) {
+      Serial.print("Error: ");
+      Serial.println(key);
+      break;
+    }
+  }
+  Serial.print("Query millis: ");
+  Serial.println(millis() - startMillis);
+
+  Serial.println("Querying 1000 random keys with findNextValue() from interval [-5000000,0)...");
+  startMillis = millis();
+  for (int i = 0; i < 1000; ++i) {
+    key = random(-5000000, 0);
+    sprintf(str, "%ld", key);
+    rt = ucdb.findKey(str, strlen(str));
+    if (rt != KEY_NOT_FOUND) {
+      Serial.print("Error: ");
+      Serial.println(str);
+      break;
+    }
+    rt = ucdb.findNextValue();
+    if (rt != KEY_NOT_FOUND) {
+      Serial.print("Error: ");
+      Serial.println(key);
+      break;
+    }
+  }
+  Serial.print("Query millis: ");
+  Serial.println(millis() - startMillis);
+
+
+  ucdb.close();
+  while (Serial.available()) {
+    Serial.read();
+  }
+}

library.properties

@@ -1,5 +1,5 @@
 name=uCDB
-version=0.4.1
+version=0.4.2
 author=Ioulianos Kakoulidis
 maintainer=Ioulianos Kakoulidis <[email protected]>
 sentence=API for querying Constant DataBase file store.

src/uCDB.cpp

@@ -16,14 +16,16 @@
 static unsigned long unpack(const byte *buff);
 
 uCDB::uCDB() {
+  zero();  
   state = CDB_CLOSED;
 }
 
 cdbResult uCDB::open(const char *fileName, unsigned long (*userHashFunc)(const void *key, unsigned long keyLen)) {
   unsigned long htPos;
   unsigned long htSlotsNum;
   byte buff[CDB_DESCRIPTOR_SIZE];
-
+
+  zero();
   cdb.close(); // Close previously opened CDB file
 
   if (!SD.exists(fileName)) {
@@ -56,10 +58,10 @@ cdbResult uCDB::open(const char *fileName, unsigned long (*userHashFunc)(const v
       continue; // Empty hash table
     }
     if ((htPos < CDB_HEADER_SIZE) || (htPos > cdb.size())) {
-      return (state = CDB_ERROR);
+      return (state = CDB_ERROR); // Critical CDB format or data integrity error          
     }
     if (((cdb.size() - htPos) >> 3) < htSlotsNum) {
-      return (state = CDB_ERROR);
+      return (state = CDB_ERROR); // Critical CDB format or data integrity error          
     }
 
     // Adjust data end position and total slots number
@@ -69,12 +71,12 @@ cdbResult uCDB::open(const char *fileName, unsigned long (*userHashFunc)(const v
     slotsNum += htSlotsNum;
 
     if (((cdb.size() - dataEndPos) >> 3) < slotsNum) {
-      return (state = CDB_ERROR);
+      return (state = CDB_ERROR); // Critical CDB format or data integrity error          
     }
   }
   // Check total
   if ((cdb.size() - dataEndPos) != 8 * slotsNum){
-    return (state = CDB_ERROR);  
+    return (state = CDB_ERROR); // Critical CDB format or data integrity error           
   }
 
   hashFunc = userHashFunc;
@@ -84,11 +86,11 @@ cdbResult uCDB::open(const char *fileName, unsigned long (*userHashFunc)(const v
 cdbResult uCDB::findKey(const void *key, unsigned long keyLen) {
   byte buff[CDB_DESCRIPTOR_SIZE];
 
+  zero();
   // Check CDB state
   switch (state) {
     case CDB_CLOSED:
     case CDB_NOT_FOUND:
-    case FILE_ERROR:
     case CDB_ERROR:
       return state;
     default:
@@ -114,42 +116,42 @@ cdbResult uCDB::findKey(const void *key, unsigned long keyLen) {
 
 cdbResult uCDB::findNextValue() {
   byte buff[CDB_BUFF_SIZE];
+  bool rd;
 
   // Check CDB state
   switch (state) {
     case CDB_CLOSED:
     case CDB_NOT_FOUND:
-    case FILE_ERROR:
     case CDB_ERROR:
       return state;
     default:
       ;
   }
 
   while (slotsToScan) {
-    if (!readDescriptor(buff, nextSlotPos)) {
-      return (state = FILE_ERROR);
-    }
-
+    rd = readDescriptor(buff, nextSlotPos);  
     // Adjust slotsToScan and next slot position
     --slotsToScan;
     nextSlotPos += CDB_DESCRIPTOR_SIZE;
     if (nextSlotPos == hashTabEndPos) {
       nextSlotPos = hashTabStartPos;
     }
+
+    if (!rd) {
+      return (state = FILE_ERROR);
+    }
 
     slotHash = unpack(buff);
     dataPos = unpack(buff + 4);
 
     if (!dataPos) {
-      slotsToScan = 0;
-      nextSlotPos = 0;
+      zero();
       return (state = KEY_NOT_FOUND);
     }
 
     // Check data position
     if ((dataPos < CDB_HEADER_SIZE) || (dataPos > (dataEndPos - CDB_DESCRIPTOR_SIZE))) {
-      return (state = CDB_ERROR);
+      return (state = CDB_ERROR); // Critical CDB format or data integrity error          
     }
 
     if (slotHash == keyHash) {
@@ -163,24 +165,29 @@ cdbResult uCDB::findNextValue() {
       //> key, value length check
       unsigned long t = dataPos + CDB_DESCRIPTOR_SIZE;
       if ((dataEndPos - t) < dataKeyLen) {
-        return (state = CDB_ERROR);          
+        return (state = CDB_ERROR); // Critical CDB format or data integrity error          
       }
       t += dataKeyLen;
       if ((dataEndPos - t) < dataValueLen) {
-        return (state = CDB_ERROR);          
+        return (state = CDB_ERROR); // Critical CDB format or data integrity error          
       }
       //< key, value length check
-
-      valueBytesAvail = dataValueLen;
 
-      if ((keyLen_ == dataKeyLen) && (compareKey() == KEY_FOUND)) {
-        return (state = KEY_FOUND);          
+      if (keyLen_ == dataKeyLen) {
+        switch (compareKey()) {
+          case KEY_FOUND:
+            valueBytesAvail = dataValueLen;          
+            return (state = KEY_FOUND);             
+          case FILE_ERROR:
+            return (state = FILE_ERROR);             
+          default:
+            ;          
+        }
       }
     }
   }
 
-  slotsToScan = 0;
-  nextSlotPos = 0;
+  zero(); // ?
 
   return (state = KEY_NOT_FOUND);
 }
@@ -213,6 +220,7 @@ int uCDB::readValue(void *buff, unsigned int byteNum) {
 }
 
 cdbResult uCDB::close() {
+  zero();  
   cdb.close();
   return (state = CDB_CLOSED);
 }
@@ -264,6 +272,11 @@ bool uCDB::readDescriptor(byte *buff, unsigned long pos) {
   }
 }
 
+void uCDB::zero() {
+  slotsToScan = 0;
+  nextSlotPos = 0;
+}
+
 unsigned long DJBHash(const void *key, unsigned long keyLen) {
   unsigned long h = 5381;
   const byte *curr = static_cast<const byte *>(key);