1 module TrailDB;
2 
3 import std.algorithm;
4 import std.conv;
5 import std.datetime;
6 import std.path : buildPath;
7 import std.range;
8 import std.stdint;
9 import std.stdio;
10 import std.string : fromStringz, format, toStringz;
11 import std.typecons;
12 
13 import traildbc;
14 
15 immutable static BUFFER_SIZE = 1 << 18;
16 
17 alias RawUuid = ubyte[16];
18 alias HexUuid = ubyte[32];
19 
20 RawUuid hexToRaw(HexUuid hexId)
21 {
22     RawUuid rawId;
23     if(int err = tdb_uuid_raw(hexId, rawId))
24     {
25         throw new Exception("Failure to convert hex uuid to raw.\n\t"
26                             ~ cast(string)fromStringz(tdb_error_str(err)));
27     }
28     return rawId;
29 }
30 
31 HexUuid rawToHex(RawUuid rawId)
32 {
33     HexUuid hexId;
34     tdb_uuid_hex(rawId, hexId);
35     return hexId;
36 }
37 
38 /* Event in a TrailDB trail. Indexing returns immutable reference to field values. */
39 /* The reference returned should be deep copied if another trail is loaded before use. */
40 struct Event
41 {
42     void* db; // Needed to get item value
43     tdb_event* event;
44 
45     @property ulong timestamp() { return event.timestamp; }
46 
47     string opIndex(ulong i)
48     {
49         uint64_t valueLength;
50         auto ret = tdb_get_item_value(db, (cast(tdb_item*)(event.items))[i], &valueLength);
51         return cast(string)ret[0 .. valueLength];
52     }
53 }
54 
55 /* D Range representing trail of events */
56 struct Trail
57 {
58     void* db;
59     void* cursor;
60 
61     int opApply(int delegate(ref Event) foreach_body)
62     {
63         tdb_event* event;
64         while((event = tdb_cursor_next(cursor)) != null)
65         {
66             Event e = Event(db, event);
67             if(int result = foreach_body(e))
68             {
69                 return result;
70             }
71         }
72 
73         return 0;
74     }
75 }
76 
77 class TrailDB
78 {
79     void* db;
80     void* cursor;
81     bool open = false;
82 
83     immutable uint64_t numTrails;
84     immutable uint64_t numEvents;
85     immutable uint64_t numFields;
86     immutable uint64_t minTimestamp;
87     immutable uint64_t maxTimestamp;
88     immutable string[] fieldNames;
89     immutable uint64_t vers;
90 
91     this(string db_path)
92     {
93         db = tdb_init();
94         if(int err = tdb_open(db, toStringz(db_path)))
95         {
96             throw new Exception("Failure to open traildb " ~ db_path ~ ".\n\t"
97                                 ~ cast(string)fromStringz(tdb_error_str(err)));
98         }
99 
100         open = true;
101 
102         numTrails     = tdb_num_trails(db);
103         numEvents     = tdb_num_events(db);
104         numFields     = tdb_num_fields(db);
105         minTimestamp  = tdb_min_timestamp(db);
106         maxTimestamp  = tdb_max_timestamp(db);
107         fieldNames    = cast(immutable)map!(i => cast(string)fromStringz(tdb_get_field_name(db, cast(uint)i)))(iota(0, numFields)).array();
108 
109         vers   = tdb_version(db);
110         cursor = tdb_cursor_new(db);
111     }
112 
113     ~this()
114     {
115         close();
116     }
117 
118     void close()
119     {
120         if(open)
121         {
122             tdb_close(db);
123             tdb_cursor_free(cursor);
124             open = false;
125         }
126     }
127 
128     ulong fieldLexiconSize(uint field)
129     {
130         return tdb_lexicon_size(db, field);
131     }
132 
133     /* Returns trail of events (a D Range)*/
134     Trail opIndex(ulong trailIndex)
135     {
136         tdb_get_trail(cursor, trailIndex);
137         return Trail(db, cursor);
138     }
139 
140     Trail opIndex(HexUuid uuid)
141     {
142         return opIndex(uuidIndex(hexToRaw(uuid)));
143     }
144 
145     Trail opIndex(RawUuid uuid)
146     {
147         return opIndex(uuidIndex(uuid));
148     }
149 
150     int opApply(int delegate(ref Trail) foreach_body)
151     {
152         tdb_willneed(db);
153         scope(exit) tdb_dontneed(db);
154 
155         foreach(i; 0 .. numTrails)
156         {
157             Trail t = opIndex(i);
158             if(int result = foreach_body(t))
159             {
160                 return result;
161             }
162         }
163 
164         return 0;
165     }
166 
167     RawUuid indexUuid(ulong index)
168     {
169         auto uuidPtr = tdb_get_uuid(db, index);
170         if(uuidPtr == null)
171         {
172             throw new Exception("No trail with index " ~ to!string(index) ~ " found.");
173         }
174         RawUuid uuid = uuidPtr[0 .. 16];
175         return uuid;
176     }
177 
178     long uuidIndex(RawUuid uuid)
179     {
180         ulong index;
181         if(int err = tdb_get_trail_id(db, uuid, &index))
182         {
183             throw new Exception("No trail with uuid " ~ cast(string)(rawToHex(uuid)) ~ " found.");
184         }
185 
186         return index;
187     }
188 }
189 
190 class TrailDBConstructor
191 {
192     void* cons;
193     string name;
194     bool finalized = false;
195     ulong[] lengthBuffer;
196     char*[] valuePointersBuffer;
197 
198     this(string name_, string[] fields)
199     {
200         lengthBuffer.length = fields.length;
201         valuePointersBuffer.length = fields.length;
202         name = name_;
203         cons = tdb_cons_init();
204         // Retain pointer to avoid GC madness
205         char*   namePtr = cast(char*)toStringz(name);
206         char*[] fieldsPtrs = cast(char*[])fields.map!(s => toStringz(s)).array();
207         if(int err = tdb_cons_open(cons, namePtr, cast(char**)fieldsPtrs, fields.length))
208         {
209             throw new Exception("Failure to open traildb constructor" ~ name ~ ".\n\t"
210                                 ~ cast(string)fromStringz(tdb_error_str(err)));
211         }
212     }
213 
214     void append(TrailDB db)
215     {
216         if(int err = tdb_cons_append(cons, db.db))
217         {
218             throw new Exception("Failure to finalize traildb constructor" ~ name ~ ".\n\t"
219                                 ~ cast(string)fromStringz(tdb_error_str(err)));
220         }
221     }
222 
223     void add(RawUuid uuid, ulong timestamp, string[] values)
224     {
225         ulong i = 0;
226         foreach(length; values.map!(v => v.length))
227         {
228             lengthBuffer[i++] = length;
229         }
230         i = 0;
231         foreach(pointer; values.map!(v => v.ptr))
232         {
233             valuePointersBuffer[i++] = cast(char*)pointer;
234         }
235 
236         if(int err = tdb_cons_add(cons, uuid, timestamp, cast(const char**)valuePointersBuffer, cast(const ulong*)lengthBuffer))
237         {
238             throw new Exception("Failure to finalize traildb constructor" ~ name ~ ".\n\t"
239                                 ~ cast(string)fromStringz(tdb_error_str(err)));
240         }
241     }
242 
243     void finalize()
244     {
245         if(!finalized)
246         {
247             if(int err = tdb_cons_finalize(cons))
248             {
249                 throw new Exception("Failure to finalize traildb constructor" ~ name ~ ".\n\t"
250                                     ~ cast(string)fromStringz(tdb_error_str(err)));
251             }
252             finalized = true;
253         }
254     }
255 
256     // Can't throw in destructor
257     ~this()
258     {
259         if(!finalized)
260         {
261             if(int err = tdb_cons_finalize(cons))
262             {
263                 writeln("Failure to finalize traildb constructor" ~ name ~ ".\n\t"
264                          ~ cast(string)fromStringz(tdb_error_str(err)));
265             }
266         }
267 
268         tdb_cons_close(cons);
269     }
270 }