Skip to content

Commit 00b26c3

Browse files
igorcanadiigorcanadi
committed
JSONDocument
Summary: After evaluating options for JSON storage, I decided to implement our own. The reason is that we'll be able to optimize it better and we get to reduce unnecessary dependencies (which is what we'd get with folly). I also plan to write a serializer/deserializer for JSONDocument with our own binary format similar to BSON. That way we'll store binary JSON format in RocksDB instead of the plain-text JSON. This means less storage and faster deserialization. There are still some inefficiencies left here. I plan to optimize them after we develop a functioning DocumentDB. That way we can move and iterate faster. Test Plan: added a unit test Reviewers: dhruba, haobo, sdong, ljin, yhchiang Reviewed By: haobo Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D18831
1 parent 9fe87b1 commit 00b26c3

File tree

5 files changed

+915
-0
lines changed

5 files changed

+915
-0
lines changed

‎Makefile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ TESTS = \
104104
stringappend_test \
105105
ttl_test \
106106
backupable_db_test \
107+
json_document_test \
107108
version_edit_test \
108109
version_set_test \
109110
file_indexer_test \
@@ -343,6 +344,9 @@ prefix_test: db/prefix_test.o $(LIBOBJECTS) $(TESTHARNESS)
343344
backupable_db_test: utilities/backupable/backupable_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
344345
$(CXX) utilities/backupable/backupable_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
345346

347+
json_document_test: utilities/document/json_document_test.o $(LIBOBJECTS) $(TESTHARNESS)
348+
$(CXX) utilities/document/json_document_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
349+
346350
ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
347351
$(CXX) utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
348352

‎include/utilities/json_document.h

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
2+
// This source code is licensed under the BSD-style license found in the
3+
// LICENSE file in the root directory of this source tree. An additional grant
4+
// of patent rights can be found in the PATENTS file in the same directory.
5+
#pragma once
6+
#ifndef ROCKSDB_LITE
7+
8+
#include <string>
9+
#include <map>
10+
#include <unordered_map>
11+
#include <vector>
12+
13+
#include "rocksdb/slice.h"
14+
15+
// We use JSONDocument for DocumentDB API
16+
// Implementation inspired by folly::dynamic and rapidjson
17+
18+
namespace rocksdb {
19+
20+
// NOTE: none of this is thread-safe
21+
class JSONDocument {
22+
public:
23+
// return nullptr on parse failure
24+
static JSONDocument* ParseJSON(const char* json);
25+
26+
enum Type {
27+
kNull,
28+
kArray,
29+
kBool,
30+
kDouble,
31+
kInt64,
32+
kObject,
33+
kString,
34+
};
35+
36+
JSONDocument(); // null
37+
/* implicit */ JSONDocument(bool b);
38+
/* implicit */ JSONDocument(double d);
39+
/* implicit */ JSONDocument(int64_t i);
40+
/* implicit */ JSONDocument(const std::string& s);
41+
/* implicit */ JSONDocument(const char* s);
42+
// constructs JSONDocument of specific type with default value
43+
explicit JSONDocument(Type type);
44+
45+
// copy constructor
46+
JSONDocument(const JSONDocument& json_document);
47+
48+
~JSONDocument();
49+
50+
Type type() const;
51+
52+
// REQUIRES: IsObject()
53+
bool Contains(const std::string& key) const;
54+
// Returns nullptr if !Contains()
55+
// don't delete the returned pointer
56+
// REQUIRES: IsObject()
57+
const JSONDocument* Get(const std::string& key) const;
58+
// REQUIRES: IsObject()
59+
JSONDocument& operator[](const std::string& key);
60+
// REQUIRES: IsObject()
61+
const JSONDocument& operator[](const std::string& key) const;
62+
// returns `this`, so you can chain operations.
63+
// Copies value
64+
// REQUIRES: IsObject()
65+
JSONDocument* Set(const std::string& key, const JSONDocument& value);
66+
67+
// REQUIRES: IsArray() == true || IsObject() == true
68+
size_t Count() const;
69+
70+
// REQUIRES: IsArray()
71+
const JSONDocument* GetFromArray(size_t i) const;
72+
// REQUIRES: IsArray()
73+
JSONDocument& operator[](size_t i);
74+
// REQUIRES: IsArray()
75+
const JSONDocument& operator[](size_t i) const;
76+
// returns `this`, so you can chain operations.
77+
// Copies the value
78+
// REQUIRES: IsArray() && i < Count()
79+
JSONDocument* SetInArray(size_t i, const JSONDocument& value);
80+
// REQUIRES: IsArray()
81+
JSONDocument* PushBack(const JSONDocument& value);
82+
83+
bool IsNull() const;
84+
bool IsArray() const;
85+
bool IsBool() const;
86+
bool IsDouble() const;
87+
bool IsInt64() const;
88+
bool IsObject() const;
89+
bool IsString() const;
90+
91+
// REQUIRES: IsBool() == true
92+
bool GetBool() const;
93+
// REQUIRES: IsDouble() == true
94+
double GetDouble() const;
95+
// REQUIRES: IsInt64() == true
96+
int64_t GetInt64() const;
97+
// REQUIRES: IsString() == true
98+
const std::string& GetString() const;
99+
100+
bool operator==(const JSONDocument& rhs) const;
101+
102+
private:
103+
class ItemsIteratorGenerator;
104+
105+
public:
106+
// REQUIRES: IsObject()
107+
ItemsIteratorGenerator Items() const;
108+
109+
// appends serialized object to dst
110+
void Serialize(std::string* dst) const;
111+
// returns nullptr if Slice doesn't represent valid serialized JSONDocument
112+
static JSONDocument* Deserialize(const Slice& src);
113+
114+
private:
115+
void SerializeInternal(std::string* dst, bool type_prefix) const;
116+
// returns false if Slice doesn't represent valid serialized JSONDocument.
117+
// Otherwise, true
118+
bool DeserializeInternal(Slice* input);
119+
120+
typedef std::vector<JSONDocument*> Array;
121+
typedef std::unordered_map<std::string, JSONDocument*> Object;
122+
123+
// iteration on objects
124+
class const_item_iterator {
125+
public:
126+
typedef Object::const_iterator It;
127+
typedef Object::value_type value_type;
128+
/* implicit */ const_item_iterator(It it) : it_(it) {}
129+
It& operator++() { return ++it_; }
130+
bool operator!=(const const_item_iterator& other) {
131+
return it_ != other.it_;
132+
}
133+
value_type operator*() { return *it_; }
134+
135+
private:
136+
It it_;
137+
};
138+
class ItemsIteratorGenerator {
139+
public:
140+
/* implicit */ ItemsIteratorGenerator(const Object& object)
141+
: object_(object) {}
142+
const_item_iterator begin() { return object_.begin(); }
143+
const_item_iterator end() { return object_.end(); }
144+
145+
private:
146+
const Object& object_;
147+
};
148+
149+
union Data {
150+
Data() : n(nullptr) {}
151+
~Data() {}
152+
153+
void* n;
154+
Array a;
155+
bool b;
156+
double d;
157+
int64_t i;
158+
std::string s;
159+
Object o;
160+
} data_;
161+
const Type type_;
162+
163+
// Our serialization format's first byte specifies the encoding version. That
164+
// way, we can easily change our format while providing backwards
165+
// compatibility. This constant specifies the current version of the
166+
// serialization format
167+
static const char kSerializationFormatVersion;
168+
};
169+
170+
} // namespace rocksdb
171+
172+
#endif // ROCKSDB_LITE

‎util/coding.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ extern void PutLengthPrefixedSliceParts(std::string* dst,
3838

3939
// Standard Get... routines parse a value from the beginning of a Slice
4040
// and advance the slice past the parsed value.
41+
extern bool GetFixed64(Slice* input, uint64_t* value);
4142
extern bool GetVarint32(Slice* input, uint32_t* value);
4243
extern bool GetVarint64(Slice* input, uint64_t* value);
4344
extern bool GetLengthPrefixedSlice(Slice* input, Slice* result);
@@ -228,6 +229,15 @@ inline int VarintLength(uint64_t v) {
228229
return len;
229230
}
230231

232+
inline bool GetFixed64(Slice* input, uint64_t* value) {
233+
if (input->size() < sizeof(uint64_t)) {
234+
return false;
235+
}
236+
*value = DecodeFixed64(input->data());
237+
input->remove_prefix(sizeof(uint64_t));
238+
return true;
239+
}
240+
231241
inline bool GetVarint32(Slice* input, uint32_t* value) {
232242
const char* p = input->data();
233243
const char* limit = p + input->size();

0 commit comments

Comments
 (0)