all repos — litestore @ b333a2b292f00764616533df73ef68a546abd30b

A minimalist nosql document store.

Implemented all method for docs except PATCH.
h3rald h3rald@h3rald.com
Sat, 31 Jan 2015 15:19:06 +0100
commit

b333a2b292f00764616533df73ef68a546abd30b

parent

dc7d7fcb5e7b09550420aab71ac3f787c78725b2

6 files changed, 416 insertions(+), 36 deletions(-)

jump to
M api_v1.nimapi_v1.nim

@@ -1,4 +1,4 @@

-import asynchttpserver, asyncdispatch, strutils, cgi, strtabs, pegs +import asynchttpserver2, asyncdispatch, strutils, cgi, strtabs, pegs, json import types, core, utils # Helper procs

@@ -45,6 +45,25 @@ var fragments = querystring.split('&')

for f in fragments: f.parseQueryOption(options) + +proc validate(req: Request, LS: LiteStore, id: string, cb: proc(req: Request, LS: LiteStore, id: string):Response): Response = + if req.reqMethod == "POST" or req.reqMethod == "PUT" or req.reqMethod == "PATCH": + var ct = "" + let body = req.body.strip + if body == "": + return resError(Http400, "Bad request: No content specified for document.") + if req.headers.hasKey("Content-type"): + ct = req.headers["Content-type"] + case ct: + of "application/json": + try: + discard body.parseJson() + except: + return resError(Http400, "Invalid JSON content - $1" % getCurrentExceptionMsg()) + else: + discard + return cb(req, LS, id) + # Low level procs proc getRawDocument(LS: LiteStore, id: string, options = newQueryOptions()): Response =

@@ -65,6 +84,22 @@ result.headers = doc.contenttype.ctHeader

result.content = doc.data result.code = Http200 +proc deleteDocument(LS: LiteStore, id: string): Response = + let doc = LS.store.retrieveDocument(id) + if doc.data == "": + result = resDocumentNotFound(id) + else: + try: + let res = LS.store.destroyDocument(id) + if res == 0: + result = resError(Http500, "Unable to delete document '$1'" % id) + else: + result.headers = {"Content-Length": "0"}.newStringTable + result.content = "" + result.code = Http204 + except: + result = resError(Http500, "Unable to delete document '$1'" % id) + proc getRawDocuments(LS: LiteStore, options = newQueryOptions()): Response = let docs = LS.store.retrieveRawDocuments(options) if docs == "[]":

@@ -74,17 +109,53 @@ result.headers = ctJsonHeader()

result.content = docs result.code = Http200 +proc postDocument(LS: LiteStore, body: string, ct: string): Response = + try: + var doc = LS.store.createDocument("", body, ct) + if doc != "": + result.headers = ctJsonHeader() + result.content = doc + result.code = Http201 + else: + result = resError(Http500, "Unable to create document.") + except: + result = resError(Http500, "Unable to create document.") + +proc putDocument(LS: LiteStore, id: string, body: string, ct: string): Response = + let doc = LS.store.retrieveDocument(id) + if doc.data == "": + # Create a new document + var doc = LS.store.createDocument(id, body, ct) + if doc != "": + result.headers = ctJsonHeader() + result.content = doc + result.code = Http201 + else: + result = resError(Http500, "Unable to create document.") + else: + # Update existing document + try: + var doc = LS.store.updateDocument(id, body, ct) + if doc != "": + result.headers = ctJsonHeader() + result.content = doc + result.code = Http200 + else: + result = resError(Http500, "Unable to update document '$1'." % id) + except: + result = resError(Http500, "Unable to update document '$1'." % id) + # Main routing proc options(req: Request, LS: LiteStore, id = ""): Response = if id != "": - result.code = Http202 + result.code = Http204 result.content = "" result.headers = {"Allow": "HEAD,GET,PUT,PATCH,DELETE"}.newStringTable else: - result.code = Http202 + result.code = Http204 result.content = "" - result.headers = {"Allow": "HEAD,GET,POST,DELETE"}.newStringTable + result.headers = {"Allow": "HEAD,GET,POST"}.newStringTable proc head(req: Request, LS: LiteStore, id = ""): Response = var options = newQueryOptions()

@@ -112,13 +183,43 @@ return LS.getRawDocuments(options)

except: return resError(Http400, "Bad request - $1" % getCurrentExceptionMsg()) +proc post(req: Request, LS: LiteStore, id = ""): Response = + if id == "": + var ct = "text/plain" + if req.headers.hasKey("Content-type"): + ct = req.headers["Content-type"] + return LS.postDocument(req.body.strip, ct) + else: + return resError(Http400, "Bad request: document ID cannot be specified in POST requests.") + +proc put(req: Request, LS: LiteStore, id = ""): Response = + if id != "": + var ct = "text/plain" + if req.headers.hasKey("Content-type"): + ct = req.headers["Content-type"] + return LS.putDocument(id, req.body.strip, ct) + else: + return resError(Http400, "Bad request: document ID must be specified in PUT requests.") + +proc delete(req: Request, LS: LiteStore, id = ""): Response = + if id != "": + return LS.deleteDocument(id) + else: + return resError(Http400, "Bad request: document ID must be specified in DELETE requests.") + proc route*(req: Request, LS: LiteStore, id = ""): Response = case req.reqMethod: + of "POST": + return validate(req, LS, id, post) + of "PUT": + return validate(req, LS, id, put) + of "DELETE": + return validate(req, LS, id, delete) of "HEAD": - return req.head(LS, id) + return validate(req, LS, id, head) of "OPTIONS": - return req.options(LS, id) + return validate(req, LS, id, options) of "GET": - return req.get(LS, id) + return validate(req, LS, id, get) else: return resError(Http405, "Method not allowed: $1" % req.reqMethod)
A asynchttpserver2.nim

@@ -0,0 +1,275 @@

+# +# Nim's Runtime Library +# (c) Copyright 2014 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +# Modified by Fabio Cevasco to allow simple processing of PUT and PATCH methods +# + +## This module implements a high performance asynchronous HTTP server. +## +## Examples +## -------- +## +## This example will create an HTTP server on port 8080. The server will +## respond to all requests with a ``200 OK`` response code and "Hello World" +## as the response body. +## +## .. code-block::nim +## var server = newAsyncHttpServer() +## proc cb(req: Request) {.async.} = +## await req.respond(Http200, "Hello World") +## +## asyncCheck server.serve(Port(8080), cb) +## runForever() + +import strtabs, asyncnet, asyncdispatch, parseutils, uri, strutils +type + Request* = object + client*: AsyncSocket # TODO: Separate this into a Response object? + reqMethod*: string + headers*: StringTableRef + protocol*: tuple[orig: string, major, minor: int] + url*: Uri + hostname*: string ## The hostname of the client that made the request. + body*: string + + AsyncHttpServer* = ref object + socket: AsyncSocket + reuseAddr: bool + + HttpCode* = enum + Http100 = "100 Continue", + Http101 = "101 Switching Protocols", + Http200 = "200 OK", + Http201 = "201 Created", + Http202 = "202 Accepted", + Http204 = "204 No Content", + Http205 = "205 Reset Content", + Http206 = "206 Partial Content", + Http300 = "300 Multiple Choices", + Http301 = "301 Moved Permanently", + Http302 = "302 Found", + Http303 = "303 See Other", + Http304 = "304 Not Modified", + Http305 = "305 Use Proxy", + Http307 = "307 Temporary Redirect", + Http400 = "400 Bad Request", + Http401 = "401 Unauthorized", + Http403 = "403 Forbidden", + Http404 = "404 Not Found", + Http405 = "405 Method Not Allowed", + Http406 = "406 Not Acceptable", + Http407 = "407 Proxy Authentication Required", + Http408 = "408 Request Timeout", + Http409 = "409 Conflict", + Http410 = "410 Gone", + Http411 = "411 Length Required", + Http418 = "418 I'm a teapot", + Http500 = "500 Internal Server Error", + Http501 = "501 Not Implemented", + Http502 = "502 Bad Gateway", + Http503 = "503 Service Unavailable", + Http504 = "504 Gateway Timeout", + Http505 = "505 HTTP Version Not Supported" + + HttpVersion* = enum + HttpVer11, + HttpVer10 + +{.deprecated: [TRequest: Request, PAsyncHttpServer: AsyncHttpServer, + THttpCode: HttpCode, THttpVersion: HttpVersion].} + +proc `==`*(protocol: tuple[orig: string, major, minor: int], + ver: HttpVersion): bool = + let major = + case ver + of HttpVer11, HttpVer10: 1 + let minor = + case ver + of HttpVer11: 1 + of HttpVer10: 0 + result = protocol.major == major and protocol.minor == minor + +proc newAsyncHttpServer*(reuseAddr = true): AsyncHttpServer = + ## Creates a new ``AsyncHttpServer`` instance. + new result + result.reuseAddr = reuseAddr + +proc addHeaders(msg: var string, headers: StringTableRef) = + for k, v in headers: + msg.add(k & ": " & v & "\c\L") + +proc sendHeaders*(req: Request, headers: StringTableRef): Future[void] = + ## Sends the specified headers to the requesting client. + var msg = "" + addHeaders(msg, headers) + return req.client.send(msg) + +proc respond*(req: Request, code: HttpCode, + content: string, headers = newStringTable()) {.async.} = + ## Responds to the request with the specified ``HttpCode``, headers and + ## content. + ## + ## This procedure will **not** close the client socket. + var customHeaders = headers + customHeaders["Content-Length"] = $content.len + var msg = "HTTP/1.1 " & $code & "\c\L" + msg.addHeaders(customHeaders) + await req.client.send(msg & "\c\L" & content) + +proc newRequest(): Request = + result.headers = newStringTable(modeCaseInsensitive) + result.hostname = "" + result.body = "" + +proc parseHeader(line: string): tuple[key, value: string] = + var i = 0 + i = line.parseUntil(result.key, ':') + inc(i) # skip : + i += line.skipWhiteSpace(i) + i += line.parseUntil(result.value, {'\c', '\L'}, i) + +proc parseProtocol(protocol: string): tuple[orig: string, major, minor: int] = + var i = protocol.skipIgnoreCase("HTTP/") + if i != 5: + raise newException(ValueError, "Invalid request protocol. Got: " & + protocol) + result.orig = protocol + i.inc protocol.parseInt(result.major, i) + i.inc # Skip . + i.inc protocol.parseInt(result.minor, i) + +proc sendStatus(client: AsyncSocket, status: string): Future[void] = + client.send("HTTP/1.1 " & status & "\c\L") + +proc processClient(client: AsyncSocket, address: string, + callback: proc (request: Request): + Future[void] {.closure, gcsafe.}) {.async.} = + while not client.isClosed: + # GET /path HTTP/1.1 + # Header: val + # \n + var request = newRequest() + request.hostname = address + assert client != nil + request.client = client + + # First line - GET /path HTTP/1.1 + let line = await client.recvLine() # TODO: Timeouts. + if line == "": + client.close() + return + let lineParts = line.split(' ') + if lineParts.len != 3: + await request.respond(Http400, "Invalid request. Got: " & line) + continue + + let reqMethod = lineParts[0] + let path = lineParts[1] + let protocol = lineParts[2] + + # Headers + var i = 0 + while true: + i = 0 + let headerLine = await client.recvLine() + if headerLine == "": + client.close(); return + if headerLine == "\c\L": break + # TODO: Compiler crash + #let (key, value) = parseHeader(headerLine) + let kv = parseHeader(headerLine) + request.headers[kv.key] = kv.value + + request.reqMethod = reqMethod + request.url = parseUri(path) + try: + request.protocol = protocol.parseProtocol() + except ValueError: + asyncCheck request.respond(Http400, "Invalid request protocol. Got: " & + protocol) + continue + + var nMethod = reqMethod.normalize + if nMethod == "post" or nMethod == "put" or nMethod == "patch": + # Check for Expect header + if request.headers.hasKey("Expect"): + if request.headers["Expect"].toLower == "100-continue": + await client.sendStatus("100 Continue") + else: + await client.sendStatus("417 Expectation Failed") + + # Read the body + # - Check for Content-length header + if request.headers.hasKey("Content-Length"): + var contentLength = 0 + if parseInt(request.headers["Content-Length"], contentLength) == 0: + await request.respond(Http400, "Bad Request. Invalid Content-Length.") + else: + request.body = await client.recv(contentLength) + assert request.body.len == contentLength + else: + await request.respond(Http400, "Bad Request. No Content-Length.") + continue + + case reqMethod.normalize + of "get", "post", "head", "put", "delete", "trace", "options", "connect", "patch": + await callback(request) + else: + await request.respond(Http400, "Invalid request method. Got: " & reqMethod) + + # Persistent connections + if (request.protocol == HttpVer11 and + request.headers["connection"].normalize != "close") or + (request.protocol == HttpVer10 and + request.headers["connection"].normalize == "keep-alive"): + # In HTTP 1.1 we assume that connection is persistent. Unless connection + # header states otherwise. + # In HTTP 1.0 we assume that the connection should not be persistent. + # Unless the connection header states otherwise. + discard + else: + request.client.close() + break + +proc serve*(server: AsyncHttpServer, port: Port, + callback: proc (request: Request): Future[void] {.closure,gcsafe.}, + address = "") {.async.} = + ## Starts the process of listening for incoming HTTP connections on the + ## specified address and port. + ## + ## When a request is made by a client the specified callback will be called. + server.socket = newAsyncSocket() + if server.reuseAddr: + server.socket.setSockOpt(OptReuseAddr, true) + server.socket.bindAddr(port, address) + server.socket.listen() + + while true: + # TODO: Causes compiler crash. + #var (address, client) = await server.socket.acceptAddr() + var fut = await server.socket.acceptAddr() + asyncCheck processClient(fut.client, fut.address, callback) + #echo(f.isNil) + #echo(f.repr) + +proc close*(server: AsyncHttpServer) = + ## Terminates the async http server instance. + server.socket.close() + +when isMainModule: + proc main = + var server = newAsyncHttpServer() + proc cb(req: Request) {.async.} = + #echo(req.reqMethod, " ", req.url) + #echo(req.headers) + let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT", + "Content-type": "text/plain; charset=utf-8"} + await req.respond(Http200, "Hello World", headers.newStringTable()) + + asyncCheck server.serve(Port(5555), cb) + runForever() + main()
M core.nimcore.nim

@@ -8,6 +8,7 @@ times,

json, pegs, strtabs, + strutils, base64 import types,

@@ -48,47 +49,52 @@ raise newException(EDatastoreUnavailable, "Datastore '$1' cannot be closed." % store.path)

# Manage Documents +proc retrieveRawDocument*(store: Datastore, id: string, options: QueryOptions = newQueryOptions()): string = + var options = options + options.single = true + var select = prepareSelectDocumentsQuery(options) + var raw_document = store.db.getRow(select.sql, id) + if raw_document[0] == "": + return "" + else: + return $store.prepareJsonDocument(raw_document) + proc createDocument*(store: Datastore, id="", data = "", contenttype = "text/plain", binary = -1, searchable = 1): string = + var id = id + var contenttype = contenttype.replace(peg"""\;(.+)$""", "") # Strip charset for now var binary = checkIfBinary(binary, contenttype) var data = data if binary == 1: data = data.encode(data.len*2) if id == "": - result = $genOid() - else: - result = id + id = $genOid() # Store document - store.db.exec(SQL_INSERT_DOCUMENT, result, data, contenttype, binary, searchable, getTime().getGMTime().format("yyyy-MM-dd'T'hh:mm:ss'Z'")) - if binary == 0 and searchable == 1: + store.db.exec(SQL_INSERT_DOCUMENT, id, data, contenttype, binary, searchable, getTime().getGMTime().format("yyyy-MM-dd'T'hh:mm:ss'Z'")) + if binary <= 0 and searchable >= 0: # Add to search index - store.db.exec(SQL_INSERT_SEARCHCONTENT, result, data) - store.addDocumentSystemTags(result, contenttype) - return result + store.db.exec(SQL_INSERT_SEARCHCONTENT, id, data) + store.addDocumentSystemTags(id, contenttype) + return $store.retrieveRawDocument(id) -proc updateDocument*(store: Datastore, id: string, data: string, contenttype = "text/plain", binary = -1, searchable = 1): int64 = +proc updateDocument*(store: Datastore, id: string, data: string, contenttype = "text/plain", binary = -1, searchable = 1): string = + var contenttype = contenttype.replace(peg"""\;(.+)$""", "") # Strip charset for now var binary = checkIfBinary(binary, contenttype) var data = data if binary == 1: data = data.encode(data.len*2) - result = store.db.execAffectedRows(SQL_UPDATE_DOCUMENT, data, contenttype, binary, searchable, getTime().getGMTime().format("yyyy-MM-dd'T'hh:mm:ss'Z'"), id) - store.destroyDocumentSystemTags(id) - store.addDocumentSystemTags(id, contenttype) - store.db.exec(SQL_UPDATE_SEARCHCONTENT, data, id) + var res = store.db.execAffectedRows(SQL_UPDATE_DOCUMENT, data, contenttype, binary, searchable, getTime().getGMTime().format("yyyy-MM-dd'T'hh:mm:ss'Z'"), id) + if res > 0: + store.destroyDocumentSystemTags(id) + store.addDocumentSystemTags(id, contenttype) + store.db.exec(SQL_UPDATE_SEARCHCONTENT, data, id) + return $store.retrieveRawDocument(id) + else: + return "" proc destroyDocument*(store: Datastore, id: string): int64 = result = store.db.execAffectedRows(SQL_DELETE_DOCUMENT, id) store.db.exec(SQL_DELETE_SEARCHCONTENT, id) store.db.exec(SQL_DELETE_DOCUMENT_TAGS, id) - -proc retrieveRawDocument*(store: Datastore, id: string, options: QueryOptions = newQueryOptions()): string = - var options = options - options.single = true - var select = prepareSelectDocumentsQuery(options) - var raw_document = store.db.getRow(select.sql, id) - if raw_document[0] == "": - return "" - else: - return $store.prepareJsonDocument(raw_document) proc retrieveDocument*(store: Datastore, id: string, options: QueryOptions = newQueryOptions()): tuple[data: string, contenttype: string] = var options = options
M server.nimserver.nim

@@ -1,4 +1,4 @@

-import asynchttpserver, asyncdispatch, times, strutils, pegs, strtabs, cgi +import asynchttpserver2, asyncdispatch, times, strutils, pegs, strtabs, cgi import types, utils, api_v1 proc getReqInfo(req: Request): string =
M types.nimtypes.nim

@@ -1,6 +1,4 @@

-import db_sqlite, pegs -from asynchttpserver import HttpCode -from strtabs import StringTableRef, newStringTable +import db_sqlite, pegs, asynchttpserver2, strtabs type EDatastoreExists* = object of Exception

@@ -53,7 +51,7 @@ ^[a-zA-Z0-9_\-?~:.@#^!]+$

""" let PEG_URL* = peg""" - ^\/{(v\d+)} \/ {([^\/]+)} \/ {(.*)} + ^\/{(v\d+)} \/ {([^\/]+)} (\/ {(.+)} / \/?)$ """ const
M utils.nimutils.nim

@@ -1,4 +1,4 @@

-import json, db_sqlite, strutils, pegs, asyncdispatch, asynchttpserver +import json, db_sqlite, strutils, pegs, asyncdispatch, asynchttpserver2 import types, queries, contenttypes proc dbQuote*(s: string): string =