all repos — litestore @ 5dca363ee51a2f987686cb0e94446771594ae55b

A minimalist nosql document store.

Merge branch 'master' of github.com:h3rald/litestore
h3rald h3rald@h3rald.com
Fri, 09 Apr 2021 10:38:48 +0200
commit

5dca363ee51a2f987686cb0e94446771594ae55b

parent

e76df1dfa89b17f6b6182988642910ab5f1d9e56

M .gitignore.gitignore

@@ -19,3 +19,4 @@ *_backup

./config.json *.db-shm *.db-wal +*.nim.bak
M src/admin/md/api_docs.mdsrc/admin/md/api_docs.md

@@ -195,6 +195,20 @@ > Tip

> > If **search** is specified, each result will contain a **highlight** property with a highlighted search snippet, and a **rank** property identified the rank of the result within the search. Results will also be automatically ordered by descending rank. +##### `like` option + +If this option is specified, retrieves a single document which id matches the specified pattern. The value give to `like` option is not used so the search pattern is specified as the regular document id as a part of the path. The regular SQL wildcards `%` and `_` can be used. To escape them use `\` encoded in the URL as `%5C`. + +Example: http://127.0.0.1:9500/docs/%/api%5C_%.md?like=1&raw=true&contents=false&search=API%20v7%Required + +If finds the first markdown file which name starts with _api\__ and which contains string _API v7 Required_. + +> %tip% +> Tip +> +> Only the first matching document is returned even if there was more than one which id matched the pattern. To get the subsequent documents use `offset` option and increase its value from _1_ up. + + ##### `tags` option Retrieve only documents with matching tag(s).
M src/admin/md/usage.mdsrc/admin/md/usage.md

@@ -30,6 +30,7 @@ * **-p**, **-\-port** —Specify server port number (default: 9500).

* **-r**, **-\-readonly** — Allow only data retrieval operations. * **-s**, **-\-store** — Specify a datastore file (default: data.db) * **--system** — Set the system flag for import, export, and delete operations +* **--importTags** — During import read tags from '_tags' file and apply them to imported documents from the same directory. * **-t**, **--type** — Specify a content type for the body an operation to be executed via the execute command. * **-u**, **--uri** — Specify an uri to execute an operation through the execute command. * **-v**, **-\-version** — Display the program version.

@@ -78,6 +79,30 @@

Import all documents stored in a directory called **system** as system documents: [litestore import -d:system --system](class:cmd) + +Import all documents stored in a directory called **media** (including subdirectories): +``` ++ media + + cars + | + _tags + | + Lamborgini.jpg + | + VW.jpg + | ` BMW.jpg + + planes + | + _tags + | + 767.jpg + | + F-16.jpg + | ` B-1.jpg + ` trains + + TGV.jpg + ` Eurostar.jpg +``` + +[litestore import -d:media --importTags](class:cmd) + +Every **_tags** file contains a list of tags, one per line, which are applied to all imported documents from the same directory. In the example above all cars and planes images will be tagged on import. The trains images, not as there is not **_tags** file in the **trains** directory. + +The individual **_tags** files are also imported. When the **--importTags** option is not set the **_tags** files are ignored and not imported. #### Exporting a directory
M src/litestore.nimsrc/litestore.nim

@@ -107,7 +107,7 @@ of opRun:

LS.serve runForever() of opImport: - LS.store.importDir(LS.directory, LS.manageSystemData) + LS.store.importDir(LS.directory, LS.manageSystemData, LS.importTags) of opExport: LS.store.exportDir(LS.directory, LS.manageSystemData) of opDelete:
M src/litestorepkg/lib/api_v7.nimsrc/litestorepkg/lib/api_v7.nim

@@ -707,7 +707,9 @@ else:

return resError(Http400, "Bad request: patch operation #$1 is malformed." % $c) c.inc if apply: - if origData.len > 0 and origData != data: + # when document is not JSON the origData is not defined + # the extra check allows editing tags for non-JSON documents + if origData != nil and origData.len > 0 and origData != data: try: var doc = LS.store.updateDocument(id, data.pretty, "application/json") if doc == "":
M src/litestorepkg/lib/cli.nimsrc/litestorepkg/lib/cli.nim

@@ -18,6 +18,7 @@ directory:string = ""

readonly = false logLevel = "warn" system = false + importTags = false mount = false auth = newJNull() middleware = newStringTable()

@@ -62,6 +63,7 @@ -p, --port Specify server port number (default: 9500).

-r, --readonly Allow only data retrieval operations. -s, --store Specify a datastore file (default: data.db) --system Set the system flag for import, export, and delete operations + --tags During import read tags from '_tags' file and apply them to imported documents from the same directory. -t, --type Specify a content type for the body an operation to be executed via the execute command. -u, --uri Specify an uri to execute an operation through the execute command. -v, --version Display the program version.

@@ -153,11 +155,14 @@ authFile = val

of "config", "c": if val == "": fail(115, "Configuration file not specified.") - configuration = val.parseFile + configuration = val.parseFile() configFile = val of "mount", "m": mount = true cliSettings["mount"] = %mount + of "importTags": + importTags = true + cliSettings["importTags"] = %importTags of "version", "v": echo pkgVersion quit(0)

@@ -187,6 +192,7 @@ LS.middleware = middleware

LS.authFile = authFile LS.config = configuration LS.configFile = configFile + LS.importTags = importTags LS.mount = mount LS.execution.file = exFile LS.execution.body = exBody
M src/litestorepkg/lib/config.nimsrc/litestorepkg/lib/config.nim

@@ -1,6 +1,6 @@

const pkgName* = "litestore" - pkgVersion* = "1.9.3" + pkgVersion* = "1.9.4" pkgAuthor* = "Fabio Cevasco" pkgDescription* = "Self-contained, lightweight, RESTful document store." pkgLicense* = "MIT"
M src/litestorepkg/lib/core.nimsrc/litestorepkg/lib/core.nim

@@ -450,6 +450,17 @@ except:

eWarn() store.rollback() +proc findDocumentId*(store: Datastore, pattern: string): string = + var select = "SELECT id FROM documents WHERE id LIKE ? ESCAPE '\\' " + var raw_document = store.db.getRow(select.sql, pattern) + LOG.debug("Retrieving document '$1'" % pattern) + if raw_document[0] == "": + LOG.debug("(No Such Document)") + result = "" + else: + result = raw_document[0] + LOG.debug("Found id: $1" % result) + proc retrieveDocument*(store: Datastore, id: string, options: QueryOptions = newQueryOptions()): tuple[data: string, contenttype: string] =

@@ -485,7 +496,7 @@

proc countDocuments*(store: Datastore): int64 = return store.db.getRow(SQL_COUNT_DOCUMENTS)[0].parseInt -proc importFile*(store: Datastore, f: string, dir = "/", system = false) = +proc importFile*(store: Datastore, f: string, dir = "/", system = false): string = if not f.fileExists: raise newException(EFileNotFound, "File '$1' not found." % f) let ext = f.splitFile.ext

@@ -520,6 +531,20 @@ eWarn()

raise if singleOp: store.commit() + return d_id + +proc importTags*(store: Datastore, d_id: string, tags: openArray[string]) = + let singleOp = not LS_TRANSACTION + store.begin() + try: + for tag in tags: + store.db.exec(SQL_INSERT_TAG, tag, d_id) + except: + store.rollback() + eWarn() + raise + if singleOp: + store.commit() proc optimize*(store: Datastore) = try:

@@ -545,15 +570,27 @@ eWarn()

quit(203) quit(0) -proc importDir*(store: Datastore, dir: string, system = false) = +proc getTagsForFile*(f: string): seq[string] = + result = newSeq[string]() + let tags_file = f.splitFile.dir / "_tags" + if tags_file.fileExists: + for tag in tags_file.lines: + result.add(tag) + + +proc importDir*(store: Datastore, dir: string, system = false, importTags = false) = var files = newSeq[string]() if not dir.dirExists: raise newException(EDirectoryNotFound, "Directory '$1' not found." % dir) for f in dir.walkDirRec(): if f.dirExists: continue - if f.splitFile.name.startsWith("."): + let fileName = f.splitFile.name + if fileName.startsWith("."): # Ignore hidden files + continue + if fileName == "_tags" and not importTags: + # Ignore tags file unless the CLI flag was set continue files.add(f) # Import single files in batch

@@ -567,7 +604,11 @@ LOG.debug("Dropping column indexes...")

store.db.dropIndexes() for f in files: try: - store.importFile(f, dir, system) + let docId = store.importFile(f, dir, system) + if not system and importTags: + let tags = getTagsForFile(f) + if tags.len > 0: + store.importTags(docId, tags) cFiles.inc if (cFiles-1) mod batchSize == 0: cBatches.inc

@@ -734,7 +775,8 @@

if LS.execution.operation == "" and LS.operation == opExecute: fail(111, "--operation option not specified") - + if LS.importTags and LS.operation != opImport: + fail(116, "--importTags option alowed only for import operation.") proc updateConfig*(LS: LiteStore) = let rawConfig = LS.config.pretty if LS.configFile != "":
M src/litestorepkg/lib/server.nimsrc/litestorepkg/lib/server.nim

@@ -248,12 +248,12 @@ return result

req.route PEG_DEFAULT_URL: info.version = "v7" info.resource = matches[0] - info.id = matches[1] + info.id = matches[1].decodeUrl return req.processApiUrl(LS, info) req.route PEG_URL: info.version = matches[0] info.resource = matches[1] - info.id = matches[2] + info.id = matches[2].decodeUrl return req.processApiUrl(LS, info) raise newException(EInvalidRequest, req.getReqInfo()) except EInvalidRequest:
M src/litestorepkg/lib/types.nimsrc/litestorepkg/lib/types.nim

@@ -88,6 +88,7 @@ cliSettings*: JsonNode

directory*: string manageSystemData*: bool file*: string + importTags*: bool mount*: bool readonly*: bool appname*: string
M src/litestorepkg/lib/utils.nimsrc/litestorepkg/lib/utils.nim

@@ -107,9 +107,12 @@ tables = options.tables & @[documents_table]

result = result & options.select.join(", ") result = result & " FROM "&tables.join(", ")&" WHERE 1=1 " if options.single: - result = result & "AND id = ?" + if options.like.len > 0: + options.limit = 1 + else: + result = result & "AND id = ?" var doc_id_col: string - if options.tags.len > 0 or options.folder.len > 0: + if options.tags.len > 0 or options.folder.len > 0 or options.like.len > 0: if options.jsonFilter.len > 0 or (options.search.len > 0 and options.select[0] != "COUNT(docid)"): doc_id_col = "$1.id" % documents_table else:

@@ -124,6 +127,8 @@ if options.modifiedBefore != "":

result = result & "AND modified < \"" & $options.modifiedBefore & "\" " if options.folder.len > 0: result = result & "AND " & doc_id_col & " BETWEEN ? and ? " + if options.like.len > 0: + result = result & "AND " & doc_id_col & " LIKE ? ESCAPE '\\' " if options.tags.len > 0: result = result & options.tags.selectDocumentsByTags(doc_id_col) if options.jsonFilter.len > 0: