all repos — litestore @ 84ce577345ec173564e6e32149130cbfd74ee38f

A minimalist nosql document store.

Improvements and refactoring, implementing HTTP api for stores
h3rald h3rald@h3rald.com
Mon, 16 Mar 2020 14:37:53 +0100
commit

84ce577345ec173564e6e32149130cbfd74ee38f

parent

20cfaacca9bd1a238a912f14da87f57ffb98de29

M src/litestore.nimsrc/litestore.nim

@@ -1,18 +1,16 @@

import strutils, - strtabs, - tables, - os, uri, httpcore, - json + json, + tables import litestorepkg/lib/types, litestorepkg/lib/logger, litestorepkg/lib/utils, litestorepkg/lib/core, - litestorepkg/lib/cli, - litestorepkg/lib/server + litestorepkg/lib/server, + litestorepkg/lib/cli export types,

@@ -27,43 +25,6 @@ {.passC: "-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_JSON1".}

when defined(linux): {.passL:"-static".} -proc processAuthConfig(configuration: JsonNode, auth: var JsonNode) = - if auth == newJNull() and configuration != newJNull() and configuration.hasKey("signature"): - auth = newJObject(); - auth["access"] = newJObject(); - auth["signature"] = configuration["signature"] - for k, v in configuration["resources"].pairs: - auth["access"][k] = newJObject() - for meth, content in v.pairs: - if content.hasKey("auth"): - auth["access"][k][meth] = content["auth"] - -proc processConfigSettings(LS: var LiteStore) = - # Process config settings if present and if no cli settings are set - if LS.config != newJNull() and LS.config.hasKey("settings"): - let settings = LS.config["settings"] - let cliSettings = LS.cliSettings - if not cliSettings.hasKey("address") and settings.hasKey("address"): - LS.address = settings["address"].getStr - if not cliSettings.hasKey("port") and settings.hasKey("port"): - LS.port = settings["port"].getInt - if not cliSettings.hasKey("store") and settings.hasKey("store"): - LS.file = settings["store"].getStr - if not cliSettings.hasKey("directory") and settings.hasKey("directory"): - LS.directory = settings["directory"].getStr - if not cliSettings.hasKey("middleware") and settings.hasKey("middleware"): - let val = settings["middleware"].getStr - for file in val.walkDir(): - if file.kind == pcFile or file.kind == pcLinkToFile: - LS.middleware[file.path.splitFile[1]] = file.path.readFile() - if not cliSettings.hasKey("log") and settings.hasKey("log"): - LS.logLevel = settings["log"].getStr - setLogLevel(LS.logLevel) - if not cliSettings.hasKey("mount") and settings.hasKey("mount"): - LS.mount = settings["mount"].getBool - if not cliSettings.hasKey("readonly") and settings.hasKey("readonly"): - LS.readonly = settings["readonly"].getBool - proc executeOperation*() = let file = LS.execution.file let body = LS.execution.body

@@ -104,64 +65,6 @@ quit(0)

else: quit(resp.code.int) -proc setup*(LS: var LiteStore, open = true) = - if not LS.file.fileExists: - try: - LOG.debug("Creating datastore: ", LS.file) - LS.file.createDatastore() - except: - eWarn() - fail(200, "Unable to create datastore '$1'" % [LS.file]) - if (open): - try: - LS.store = LS.file.openDatastore() - try: - LS.store.upgradeDatastore() - except: - fail(203, "Unable to upgrade datastore '$1'" % [LS.file]) - if LS.mount: - try: - LS.store.mountDir(LS.directory) - except: - eWarn() - fail(202, "Unable to mount directory '$1'" % [LS.directory]) - except: - fail(201, "Unable to open datastore '$1'" % [LS.file]) - -proc initStore(LS: var LiteStore) = - if LS.configFile == "": - # Attempt to retrieve config.json from system documents - let options = newQueryOptions(true) - let rawDoc = LS.store.retrieveRawDocument("config.json", options) - if rawDoc != "": - LS.config = rawDoc.parseJson()["data"] - - if LS.config != newJNull(): - # Process config settings - LS.processConfigSettings() - # Process auth from config settings - processAuthConfig(LS.config, LS.auth) - - if LS.auth == newJNull(): - # Attempt to retrieve auth.json from system documents - let options = newQueryOptions(true) - let rawDoc = LS.store.retrieveRawDocument("auth.json", options) - if rawDoc != "": - LS.auth = rawDoc.parseJson()["data"] - - # Validation - if LS.directory == "" and (LS.operation in [opDelete, opImport, opExport] or LS.mount): - fail(105, "--directory option not specified.") - - if LS.execution.file == "" and (LS.execution.operation in ["put", "post", "patch"]): - fail(109, "--file option not specified") - - if LS.execution.uri == "" and LS.operation == opExecute: - fail(110, "--uri option not specified") - - if LS.execution.operation == "" and LS.operation == opExecute: - fail(111, "--operation option not specified") - # stores: { # test: { # file: 'path/to/test.db',

@@ -171,24 +74,24 @@ # signature: ''

# } # } # } -proc initStores() = +proc initStores*() = if LS.config.kind == JObject and LS.config.hasKey("stores"): for k, v in LS.config["stores"].pairs: - # TODO error handling - LSDICT[k] = initLiteStore() - LSDICT[k].file = v["file"].getStr + if not v.hasKey("file"): + fail(120, "File not specified for store '$1'" % k) + let file = v["file"].getStr + var config = newJNull() if v.hasKey("config"): - LSDICT[k].config = v["config"] - for k in LSDICT.keys: - LOG.info("Initializing store '$1'" % k) - LSDICT[k].setup(true) - LSDICT[k].initStore() - LOG.info("Initializing main store") + config = v["config"] + LSDICT[k] = LS.addStore(k, file, config) + LOG.info("Initializing master store") LS.setup(true) LS.initStore() - LSDICT["main"] = LS + LSDICT["master"] = LS when isMainModule: + + run() # Manage vacuum operation separately if LS.operation == opVacuum:

@@ -196,7 +99,6 @@ LS.setup(false)

vacuum LS.file else: # Open Datastore - #LS.setup(true) initStores() case LS.operation: of opRun:
M src/litestorepkg/lib/api_v7.nimsrc/litestorepkg/lib/api_v7.nim

@@ -358,6 +358,19 @@ else:

result.content = $doc result.code = Http200 +proc getStore*(LS: LiteStore, id: string, options = newQueryOptions(), req: LSRequest): LSResponse = + if (not LSDICT.hasKey(id)): + return resStoreNotFound(id) + let store = LSDICT[id] + var doc = newJObject() + doc["id"] = %id + doc["file"] = %store.file + doc["config"] = store.config + result.headers = ctJsonHeader() + setOrigin(LS, req, result.headers) + result.content = $doc + result.code = Http200 + proc getIndex*(LS: LiteStore, id: string, options = newQueryOptions(), req: LSRequest): LSResponse = let doc = LS.store.retrieveIndex(id, options) result.headers = ctJsonHeader()

@@ -438,6 +451,7 @@ for k, v in LSDICT.pairs:

var store = newJObject() store["id"] = %k store["file"] = %v.file + store["config"] = v.config docs.add(store) var content = newJObject() content["total"] = %LSDICT.len

@@ -543,7 +557,20 @@ LS.store.createIndex(id, field)

result.headers = ctJsonHeader() setOrigin(LS, req, result.headers) result.content = "{\"id\": \"$1\", \"field\": \"$2\"}" % [id, field] - result.code = Http200 + result.code = Http201 + except: + eWarn() + result = resError(Http500, "Unable to create index.") + +proc putStore*(LS: LiteStore, id: string, content: JsonNode, req: LSRequest): LSResponse = + try: + if (not id.match(PEG_STORE)): + return resError(Http400, "invalid store ID: $1" % id) + if (LSDICT.hasKey(id)): + return resError(Http409, "Store already exists: $1" % id) + # TODO: create store + result = getStore(LS, id, newQueryOptions(), req) + result.code = Http201 except: eWarn() result = resError(Http500, "Unable to create index.")

@@ -764,6 +791,33 @@ result.headers = newHttpHeaders(TAB_HEADERS)

setOrigin(LS, req, result.headers) result.headers["Allow"] = "HEAD, GET, OPTIONS, POST" result.headers["Access-Control-Allow-Methods"] = "HEAD, GET, OPTIONS, POST" + of "stores": + result.code = Http204 + result.content = "" + result.headers = newHttpHeaders(TAB_HEADERS) + setOrigin(LS, req, result.headers) + if id != "": + result.code = Http204 + result.content = "" + if LS.readonly: + result.headers["Allow"] = "GET, OPTIONS" + result.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" + else: + result.headers["Allow"] = "GET, OPTIONS, PUT, DELETE" + result.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS, PUT, DELETE" + else: + result.code = Http204 + result.content = "" + if LS.readonly: + result.headers = newHttpHeaders(TAB_HEADERS) + setOrigin(LS, req, result.headers) + result.headers["Allow"] = "GET, OPTIONS" + result.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" + else: + result.headers = newHttpHeaders(TAB_HEADERS) + setOrigin(LS, req, result.headers) + result.headers["Allow"] = "GET, OPTIONS" + result.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" else: discard # never happens really.

@@ -829,10 +883,10 @@ of "stores":

var options = newQueryOptions() try: parseQueryOptions(req.url.query, options); - #if id != "": - # return LS.getIndex(id, options, req) - #else: - return LS.getStores(options, req) + if id != "": + return LS.getStore(id, options, req) + else: + return LS.getStores(options, req) except: return resError(Http400, "Bad Request - $1" % getCurrentExceptionMsg()) of "info":
M src/litestorepkg/lib/cli.nimsrc/litestorepkg/lib/cli.nim

@@ -5,7 +5,7 @@ json,

os, strtabs import - logger, + core, config, types, utils

@@ -68,142 +68,128 @@ -v, --version Display the program version.

-w, --middleware Specify a path to a folder containing middleware definitions. """ -proc setLogLevel*(val: string) = - case val: - of "info": - LOG.level = lvInfo - of "warn": - LOG.level = lvWarn - of "debug": - LOG.level = lvDebug - of "error": - LOG.level = lvError - of "none": - LOG.level = lvNone - else: - fail(103, "Invalid log level '$1'" % val) - -for kind, key, val in getOpt(): - case kind: - of cmdArgument: - case key: - of "run": - operation = opRun - of "import": - operation = opImport - of "execute": - operation = opExecute - of "export": - operation = opExport - of "delete": - operation = opDelete - of "optimize": - operation = opOptimize - of "vacuum": - operation = opVacuum - else: - discard - of cmdLongOption, cmdShortOption: - case key: - of "address", "a": - if val == "": - fail(100, "Address not specified.") - address = val - cliSettings["address"] = %address - of "port", "p": - if val == "": - fail(101, "Port not specified.") - port = val.parseInt - cliSettings["port"] = %port - of "store", "s": - file = val - cliSettings["store"] = %file - of "system": - system = true - of "log", "l": - if val == "": - fail(102, "Log level not specified.") - setLogLevel(val) - logLevel = val - cliSettings["log"] = %logLevel - of "directory", "d": - if val == "": - fail(104, "Directory not specified.") - directory = val - cliSettings["directory"] = %directory - of "middleware", "w": - if val == "": - fail(115, "Middleware path not specified.") - if not val.existsDir(): - fail(116, "Middleware directory does not exist.") - for file in val.walkDir(): - if file.kind == pcFile or file.kind == pcLinkToFile: - middleware[file.path.splitFile[1]] = file.path.readFile() - cliSettings["middleware"] = %val - of "operation", "o": - if val == "": - fail(106, "Operation not specified.") - exOperation = val - of "file", "f": - if val == "": - fail(107, "File not specified.") - exFile = val - of "uri", "u": - if val == "": - fail(108, "URI not specified.") - exUri = val - of "body", "b": - if val == "": - fail(112, "Body not specified.") - exBody = val - of "type", "t": - if val == "": - fail(113, "Content type not specified.") - exType = val - of "auth": - if val == "": - fail(114, "Authentication/Authorization configuration file not specified.") - authFile = val - of "config", "c": - if val == "": - fail(115, "Configuration file not specified.") - configuration = val.parseFile - configFile = val - of "mount", "m": - mount = true - cliSettings["mount"] = %mount - of "version", "v": - echo pkgVersion - quit(0) - of "help", "h": - echo usage - quit(0) - of "readonly", "r": - readonly = true - cliSettings["readonly"] = %readonly - else: - discard - else: - discard +proc run*() = + for kind, key, val in getOpt(): + case kind: + of cmdArgument: + case key: + of "run": + operation = opRun + of "import": + operation = opImport + of "execute": + operation = opExecute + of "export": + operation = opExport + of "delete": + operation = opDelete + of "optimize": + operation = opOptimize + of "vacuum": + operation = opVacuum + else: + discard + of cmdLongOption, cmdShortOption: + case key: + of "address", "a": + if val == "": + fail(100, "Address not specified.") + address = val + cliSettings["address"] = %address + of "port", "p": + if val == "": + fail(101, "Port not specified.") + port = val.parseInt + cliSettings["port"] = %port + of "store", "s": + file = val + cliSettings["store"] = %file + of "system": + system = true + of "log", "l": + if val == "": + fail(102, "Log level not specified.") + setLogLevel(val) + logLevel = val + cliSettings["log"] = %logLevel + of "directory", "d": + if val == "": + fail(104, "Directory not specified.") + directory = val + cliSettings["directory"] = %directory + of "middleware", "w": + if val == "": + fail(115, "Middleware path not specified.") + if not val.existsDir(): + fail(116, "Middleware directory does not exist.") + for file in val.walkDir(): + if file.kind == pcFile or file.kind == pcLinkToFile: + middleware[file.path.splitFile[1]] = file.path.readFile() + cliSettings["middleware"] = %val + of "operation", "o": + if val == "": + fail(106, "Operation not specified.") + exOperation = val + of "file", "f": + if val == "": + fail(107, "File not specified.") + exFile = val + of "uri", "u": + if val == "": + fail(108, "URI not specified.") + exUri = val + of "body", "b": + if val == "": + fail(112, "Body not specified.") + exBody = val + of "type", "t": + if val == "": + fail(113, "Content type not specified.") + exType = val + of "auth": + if val == "": + fail(114, "Authentication/Authorization configuration file not specified.") + authFile = val + of "config", "c": + if val == "": + fail(115, "Configuration file not specified.") + configuration = val.parseFile + configFile = val + of "mount", "m": + mount = true + cliSettings["mount"] = %mount + of "version", "v": + echo pkgVersion + quit(0) + of "help", "h": + echo usage + quit(0) + of "readonly", "r": + readonly = true + cliSettings["readonly"] = %readonly + else: + discard + else: + discard -LS.operation = operation -LS.address = address -LS.port = port -LS.file = file -LS.directory = directory -LS.readonly = readonly -LS.favicon = favicon -LS.logLevel = logLevel -LS.cliSettings = cliSettings -LS.auth = auth -LS.manageSystemData = system -LS.middleware = middleware -LS.authFile = authFile -LS.config = configuration -LS.configFile = configFile -LS.mount = mount -LS.execution.file = exFile -LS.execution.body = exBody -LS.execution.ctype = exType -LS.execution.uri = exUri -LS.execution.operation = exOperation + LS.operation = operation + LS.address = address + LS.port = port + LS.file = file + LS.directory = directory + LS.readonly = readonly + LS.favicon = favicon + LS.logLevel = logLevel + LS.cliSettings = cliSettings + LS.auth = auth + LS.manageSystemData = system + LS.middleware = middleware + LS.authFile = authFile + LS.config = configuration + LS.configFile = configFile + LS.mount = mount + LS.execution.file = exFile + LS.execution.body = exBody + LS.execution.ctype = exType + LS.execution.uri = exUri + LS.execution.operation = exOperation
M src/litestorepkg/lib/core.nimsrc/litestorepkg/lib/core.nim

@@ -7,6 +7,7 @@ oids,

json, pegs, strtabs, + tables, strutils, base64, math

@@ -592,3 +593,129 @@ result = 0

var ids = store.db.getAllRows(SQL_SELECT_DOCUMENT_IDS_BY_TAG, tag) for id in ids: result.inc(store.destroyDocument(id[0]).int) + +proc setLogLevel*(val: string) = + case val: + of "info": + LOG.level = lvInfo + of "warn": + LOG.level = lvWarn + of "debug": + LOG.level = lvDebug + of "error": + LOG.level = lvError + of "none": + LOG.level = lvNone + else: + fail(103, "Invalid log level '$1'" % val) + +proc processAuthConfig(configuration: JsonNode, auth: var JsonNode) = + if auth == newJNull() and configuration != newJNull() and configuration.hasKey("signature"): + auth = newJObject(); + auth["access"] = newJObject(); + auth["signature"] = configuration["signature"] + for k, v in configuration["resources"].pairs: + auth["access"][k] = newJObject() + for meth, content in v.pairs: + if content.hasKey("auth"): + auth["access"][k][meth] = content["auth"] + +proc processConfigSettings(LS: var LiteStore) = + # Process config settings if present and if no cli settings are set + if LS.config != newJNull() and LS.config.hasKey("settings"): + let settings = LS.config["settings"] + let cliSettings = LS.cliSettings + if not cliSettings.hasKey("address") and settings.hasKey("address"): + LS.address = settings["address"].getStr + if not cliSettings.hasKey("port") and settings.hasKey("port"): + LS.port = settings["port"].getInt + if not cliSettings.hasKey("store") and settings.hasKey("store"): + LS.file = settings["store"].getStr + if not cliSettings.hasKey("directory") and settings.hasKey("directory"): + LS.directory = settings["directory"].getStr + if not cliSettings.hasKey("middleware") and settings.hasKey("middleware"): + let val = settings["middleware"].getStr + for file in val.walkDir(): + if file.kind == pcFile or file.kind == pcLinkToFile: + LS.middleware[file.path.splitFile[1]] = file.path.readFile() + if not cliSettings.hasKey("log") and settings.hasKey("log"): + LS.logLevel = settings["log"].getStr + setLogLevel(LS.logLevel) + if not cliSettings.hasKey("mount") and settings.hasKey("mount"): + LS.mount = settings["mount"].getBool + if not cliSettings.hasKey("readonly") and settings.hasKey("readonly"): + LS.readonly = settings["readonly"].getBool + +proc setup*(LS: var LiteStore, open = true) = + if not LS.file.fileExists: + try: + LS.file.createDatastore() + except: + eWarn() + fail(200, "Unable to create datastore '$1'" % [LS.file]) + if (open): + try: + LS.store = LS.file.openDatastore() + try: + LS.store.upgradeDatastore() + except: + fail(203, "Unable to upgrade datastore '$1'" % [LS.file]) + if LS.mount: + try: + LS.store.mountDir(LS.directory) + except: + eWarn() + fail(202, "Unable to mount directory '$1'" % [LS.directory]) + except: + fail(201, "Unable to open datastore '$1'" % [LS.file]) + +proc initStore*(LS: var LiteStore) = + if LS.configFile == "": + # Attempt to retrieve config.json from system documents + let options = newQueryOptions(true) + let rawDoc = LS.store.retrieveRawDocument("config.json", options) + if rawDoc != "": + LS.config = rawDoc.parseJson()["data"] + + if LS.config != newJNull(): + # Process config settings + LS.processConfigSettings() + # Process auth from config settings + processAuthConfig(LS.config, LS.auth) + + if LS.auth == newJNull(): + # Attempt to retrieve auth.json from system documents + let options = newQueryOptions(true) + let rawDoc = LS.store.retrieveRawDocument("auth.json", options) + if rawDoc != "": + LS.auth = rawDoc.parseJson()["data"] + + # Validation + if LS.directory == "" and (LS.operation in [opDelete, opImport, opExport] or LS.mount): + fail(105, "--directory option not specified.") + + if LS.execution.file == "" and (LS.execution.operation in ["put", "post", "patch"]): + fail(109, "--file option not specified") + + if LS.execution.uri == "" and LS.operation == opExecute: + fail(110, "--uri option not specified") + + if LS.execution.operation == "" and LS.operation == opExecute: + fail(111, "--operation option not specified") + + +proc addStore*(LS: LiteStore, id, file: string, config = newJNull()): LiteStore = + result = initLiteStore() + result.address = LS.address + result.port = LS.port + result.appname = LS.appname + result.appversion = LS.appversion + result.favicon = LS.favicon + # TODO error handling + result.file = file + if config != newJNull(): + result.config = config + LOG.info("Initializing store '$1'" % id) + result.setup(true) + result.initStore() +
M src/litestorepkg/lib/server.nimsrc/litestorepkg/lib/server.nim

@@ -263,17 +263,16 @@ let trace = e.getStackTrace()

return resError(Http500, "Internal Server Error: $1" % getCurrentExceptionMsg(), trace) -proc process*(req: LSRequest, LSDICT: Table[string, LiteStore]): LSResponse {.gcsafe.}= +proc process*(req: LSRequest, LSDICT: OrderedTable[string, LiteStore]): LSResponse {.gcsafe.}= var matches = @["", ""] if req.url.path.find(PEG_STORE_URL, matches) != -1: let id = matches[0] let path = matches[1] - if not LSDICT.hasKey(id): - return resError(Http400, "Unknown store '$1'" % id) if path == "": var info: ResourceInfo info.version = "v7" info.resource = "stores" + info.id = id return req.processApiUrl(LS, info) else: var newReq = req
M src/litestorepkg/lib/types.nimsrc/litestorepkg/lib/types.nim

@@ -232,6 +232,7 @@ var

PEG_TAG* {.threadvar.}: Peg PEG_USER_TAG* {.threadvar.}: Peg PEG_INDEX* {.threadvar}: Peg + PEG_STORE* {.threadvar}: Peg PEG_JSON_FIELD* {.threadvar.}: Peg PEG_DEFAULT_URL* {.threadvar.}: Peg PEG_STORE_URL* {.threadvar.}: Peg

@@ -240,6 +241,7 @@

PEG_TAG = peg"""^\$? [a-zA-Z0-9_\-?~:.@#^!+]+$""" PEG_USER_TAG = peg"""^[a-zA-Z0-9_\-?~:.@#^!+]+$""" PEG_INDEX = peg"""^[a-zA-Z0-9_]+$""" +PEG_STORE = peg"""^[a-zA-Z0-9_]+$""" PEG_JSON_FIELD = peg"""'$' ('.' [a-z-A-Z0-9_]+)+""" PEG_DEFAULT_URL = peg"""^\/{(docs / info / dir / tags / indexes / stores)} (\/ {(.+)} / \/?)$""" PEG_STORE_URL = peg"""^\/stores \/ {([a-z0-9_]+)} (\/ {(.+)} / \/?)$"""

@@ -247,9 +249,9 @@ PEG_URL = peg"""^\/({(v\d+)} \/) {([^\/]+)} (\/ {(.+)} / \/?)$"""

# Initialize LiteStore var LS* {.threadvar.}: LiteStore -var LSDICT* {.threadvar.}: Table[string, LiteStore] +var LSDICT* {.threadvar.}: OrderedTable[string, LiteStore] var TAB_HEADERS* {.threadvar.}: array[0..2, (string, string)] -LSDICT = initTable[string, LiteStore]() +LSDICT = initOrderedTable[string, LiteStore]() LS.appversion = pkgVersion LS.appname = appname
M src/litestorepkg/lib/utils.nimsrc/litestorepkg/lib/utils.nim

@@ -309,6 +309,9 @@

proc resIndexNotFound*(id: string): LSResponse = resError(Http404, "Index '$1' not found." % id) +proc resStoreNotFound*(id: string): LSResponse = + resError(Http404, "Store '$1' not found." % id) + proc eWarn*() = var e = getCurrentException() LOG.warn(e.msg)