all repos — litestore @ 49ecdd02163f833c379137fcfe0564e5b8fa3063

A minimalist nosql document store.

Implemented filesystem mirroring (create/update/delete docs).
h3rald h3rald@h3rald.com
Thu, 12 Mar 2015 13:20:59 +0100
commit

49ecdd02163f833c379137fcfe0564e5b8fa3063

parent

343ed9059c45739848f7c3215b4e88cde40b2ba5

8 files changed, 68 insertions(+), 17 deletions(-)

jump to
M app/index.htmlapp/index.html

@@ -37,4 +37,4 @@ <script src="js/navbar.js"> </script>

<script src="js/modules.js"> </script> <script src="js/app.js"> </script> </body> -</html> +</html>
M app/js/models.jsapp/js/models.js

@@ -2,6 +2,7 @@ (function(){

window.Page = {}; window.Info = {}; window.Doc = {}; + var u = window.LS.utils; Page.get = function(id) { var content = m.prop("");
M lib/cli.nimlib/cli.nim

@@ -18,6 +18,7 @@ operation = opRun

directory = "" readonly = false logLevel = lvlInfo + mirror = false var f = newStringStream(cfgfile) if f != nil:

@@ -66,6 +67,7 @@ --import Import the specified directory (Store all its contents).

-l, --log Specify the log level: debug, info, warn, error, fatal, none (default: info) -p, --port Specify port number (default: 9500). -r, --readonly Allow only data retrieval operations. + -m, --mirror Import the specified directory, run server and mirror database changes to filesystem. -v, --version Display the program version. """

@@ -93,6 +95,12 @@ if val == "":

fail(104, "Directory to import not specified.") operation = opImport directory = val + of "mirror", "m": + if val == "": + fail(104, "Directory to mirror not specified.") + operation = opRun + directory = val + mirror = true of "export": if val == "": fail(105, "Directory to export not specified.")

@@ -127,6 +135,7 @@ LS.appversion = version

LS.readonly = readonly LS.appname = appname LS.favicon = favicon +LS.mirror = mirror # Initialize loggers
M lib/core.nimlib/core.nim

@@ -45,8 +45,13 @@ raise newException(EDatastoreDoesNotExist, "Datastore '$1' does not exists." % file)

try: result.db = db.open(file, "", "", "") result.path = file + result.mirror = "" except: raise newException(EDatastoreUnavailable, "Datastore '$1' cannot be opened." % file) + +proc hasMirror(store: Datastore): bool = + return store.mirror.len > 0 + # Manage Documents

@@ -60,27 +65,35 @@ return ""

else: return $store.prepareJsonDocument(raw_document) -proc createDocument*(store: Datastore, id="", data = "", contenttype = "text/plain", binary = -1, searchable = 1): string = +proc createDocument*(store: Datastore, id="", rawdata = "", contenttype = "text/plain", binary = -1, searchable = 1): string = var id = id var contenttype = contenttype.replace(peg"""\;(.+)$""", "") # Strip charset for now var binary = checkIfBinary(binary, contenttype) - var data = data + var data = rawdata if binary == 1: data = data.encode(data.len*2) if id == "": id = $genOid() # Store document - store.db.exec(SQL_INSERT_DOCUMENT, id, data, contenttype, binary, searchable, currentTime()) - if binary <= 0 and searchable >= 0: - # Add to search index - store.db.exec(SQL_INSERT_SEARCHCONTENT, id, data) - store.addDocumentSystemTags(id, contenttype) + var res = store.db.execAffectedRows(SQL_INSERT_DOCUMENT, id, data, contenttype, binary, searchable, currentTime()) + if res > 0: + if binary <= 0 and searchable >= 0: + # Add to search index + store.db.exec(SQL_INSERT_SEARCHCONTENT, id, data) + store.addDocumentSystemTags(id, contenttype) + if store.hasMirror: + var filename = id.unixToNativePath + if fileExists(filename): + var file = filename.open(fmWrite) + file.write(rawdata) + else: + raise newException(EFileExists, "File already exists: $1" % filename) return $store.retrieveRawDocument(id) -proc updateDocument*(store: Datastore, id: string, data: string, contenttype = "text/plain", binary = -1, searchable = 1): string = +proc updateDocument*(store: Datastore, id: string, rawdata: string, contenttype = "text/plain", binary = -1, searchable = 1): string = var contenttype = contenttype.replace(peg"""\;(.+)$""", "") # Strip charset for now var binary = checkIfBinary(binary, contenttype) - var data = data + var data = rawdata if binary == 1: data = data.encode(data.len*2) var res = store.db.execAffectedRows(SQL_UPDATE_DOCUMENT, data, contenttype, binary, searchable, currentTime(), id)

@@ -88,6 +101,13 @@ if res > 0:

#store.destroyDocumentSystemTags(id) #store.addDocumentSystemTags(id, contenttype) store.db.exec(SQL_UPDATE_SEARCHCONTENT, data, id) + if store.hasMirror: + var filename = id.unixToNativePath + if fileExists(filename): + var file = filename.open(fmWrite) + file.write(rawdata) + else: + raise newException(EFileNotFound, "File not found: $1" % filename) return $store.retrieveRawDocument(id) else: return ""

@@ -97,8 +117,15 @@ store.db.exec(SQL_SET_DOCUMENT_MODIFIED, id, currentTime())

proc destroyDocument*(store: Datastore, id: string): int64 = result = store.db.execAffectedRows(SQL_DELETE_DOCUMENT, id) - store.db.exec(SQL_DELETE_SEARCHCONTENT, id) - store.db.exec(SQL_DELETE_DOCUMENT_TAGS, id) + if result > 0: + store.db.exec(SQL_DELETE_SEARCHCONTENT, id) + store.db.exec(SQL_DELETE_DOCUMENT_TAGS, id) + if store.hasMirror: + var filename = id.unixToNativePath + if fileExists(filename): + removeFile(id.unixToNativePath) + else: + raise newException(EFileNotFound, "File not found: $1" % filename) proc retrieveDocument*(store: Datastore, id: string, options: QueryOptions = newQueryOptions()): tuple[data: string, contenttype: string] = var options = options

@@ -166,6 +193,7 @@ tag_array.add(obj)

return %tag_array proc importDir*(store: Datastore, dir: string) = + # TODO: Only allow directory names (not paths)? if not dir.dirExists: raise newException(EDirectoryNotFound, "Directory '$1' not found." % dir) for f in dir.walkDirRec():

@@ -175,7 +203,7 @@ if f.splitFile.name.startsWith("."):

# Ignore hidden files continue let ext = f.splitFile.ext - var d_id = f + var d_id = f.replace("\\", "/") var d_contents = f.readFile var d_ct = "application/octet-stream" if CONTENT_TYPES.hasKey(ext):

@@ -191,7 +219,7 @@

proc exportDir*(store: Datastore, dir: string) = let docs = store.db.getAllRows(SQL_SELECT_DOCUMENTS_BY_TAG, "$dir:"&dir) for doc in docs: - let file = doc[0] + let file = doc[0].unixToNativePath var data: string if doc[3].parseInt == 1: data = doc[1].decode

@@ -204,6 +232,13 @@ proc deleteDir*(store: Datastore, dir: string) =

store.db.exec(SQL_DELETE_DOCUMENTS_BY_TAG, "$dir:"&dir) store.db.exec(SQL_DELETE_SEARCHCONTENTS_BY_TAG, "$dir:"&dir) store.db.exec(SQL_DELETE_TAGS_BY_TAG, "$dir:"&dir) + +proc mountDir*(store: var Datastore, dir:string) = + if not dir.dirExists: + raise newException(EDirectoryNotFound, "Directory '$1' not found." % dir) + store.deleteDir(dir) + store.importDir(dir) + store.mirror = dir proc destroyDocumentsByTag*(store: Datastore, tag: string): int64 = result = 0
M lib/server.nimlib/server.nim

@@ -53,5 +53,7 @@ info(getReqInfo(req).replace("$", "$$"))

let res = req.route(LS) await req.respond(res.code, res.content, res.headers) info(LS.appname & " v" & LS.appversion & " started on " & LS.address & ":" & $LS.port & ".") + if LS.mirror: + info("Mirroring datastore changes to: " & LS.directory) asyncCheck server.serve(LS.port.Port, handleHttpRequest, LS.address)
M lib/types.nimlib/types.nim

@@ -6,10 +6,13 @@ EDatastoreDoesNotExist* = object of Exception

EDatastoreUnavailable* = object of Exception EInvalidTag* = object of Exception EDirectoryNotFound* = object of Exception + EFileNotFound* = object of Exception + EFileExists* = object of Exception EInvalidRequest* = object of Exception Datastore* = object db*: TDbConn path*: string + mirror*: string QueryOptions* = object select*: string single*:bool

@@ -31,6 +34,7 @@ port*: int

operation*: Operation directory*: string file*: string + mirror*: bool readonly*: bool appname*: string appversion*: string
M litestore.nimlitestore.nim

@@ -30,6 +30,8 @@ except:

fail(200, "Unable to create datastore '$1'" % [LS.file]) try: LS.store = LS.file.openDatastore() + if LS.mirror: + LS.store.mountDir(LS.directory) except: fail(201, "Unable to open datastore '$1'" % [LS.file]) case LS.operation:
M run_apprun_app

@@ -1,4 +1,2 @@

#!/usr/bin/env bash -./litestore --delete:app -./litestore --import:app -./litestore +./litestore --mirror:app