all repos — litestore @ 71b51221ee98c64c3fc45cf92e3fc1a2e3df9617

A minimalist nosql document store.

When importing files when directory contains _tags file
use its content as tags for all files in that directory
kaminski kaminski@naew.nato.int
Fri, 26 Mar 2021 20:56:07 +0100
commit

71b51221ee98c64c3fc45cf92e3fc1a2e3df9617

parent

2ecc1ea94ba91fcc7e4f02d25e69729d0a913a2a

M src/admin/md/usage.mdsrc/admin/md/usage.md

@@ -30,6 +30,7 @@ * **-p**, **-\-port** —Specify server port number (default: 9500).

* **-r**, **-\-readonly** — Allow only data retrieval operations. * **-s**, **-\-store** — Specify a datastore file (default: data.db) * **--system** — Set the system flag for import, export, and delete operations +* **--importTags** — During import read tags from '_tags' file and apply them to imported documents from the same directory. * **-t**, **--type** — Specify a content type for the body an operation to be executed via the execute command. * **-u**, **--uri** — Specify an uri to execute an operation through the execute command. * **-v**, **-\-version** — Display the program version.

@@ -78,6 +79,30 @@

Import all documents stored in a directory called **system** as system documents: [litestore import -d:system --system](class:cmd) + +Import all documents stored in a directory called **media** (including subdirectories): +``` ++ media + + cars + | + _tags + | + Lamborgini.jpg + | + VW.jpg + | ` BMW.jpg + + planes + | + _tags + | + 767.jpg + | + F-16.jpg + | ` B-1.jpg + ` trains + + TGV.jpg + ` Eurostar.jpg +``` + +[litestore import -d:media --importTags](class:cmd) + +Every **_tags** file contains a list of tags, one per line, which are applied to all imported documents from the same directory. In the example above all cars and planes images will be tagged on import. The trains images, not as there is not **_tags** file in the **trains** directory. + +The individual **_tags** files are also imported. When the **--importTags** option is not set the **_tags** files are ignored and not imported. #### Exporting a directory
M src/litestore.nimsrc/litestore.nim

@@ -107,7 +107,7 @@ of opRun:

LS.serve runForever() of opImport: - LS.store.importDir(LS.directory, LS.manageSystemData) + LS.store.importDir(LS.directory, LS.manageSystemData, LS.importTags) of opExport: LS.store.exportDir(LS.directory, LS.manageSystemData) of opDelete:
M src/litestorepkg/lib/cli.nimsrc/litestorepkg/lib/cli.nim

@@ -18,6 +18,7 @@ directory:string = ""

readonly = false logLevel = "warn" system = false + importTags = false mount = false auth = newJNull() middleware = newStringTable()

@@ -62,6 +63,7 @@ -p, --port Specify server port number (default: 9500).

-r, --readonly Allow only data retrieval operations. -s, --store Specify a datastore file (default: data.db) --system Set the system flag for import, export, and delete operations + --tags During import read tags from '_tags' file and apply them to imported documents from the same directory. -t, --type Specify a content type for the body an operation to be executed via the execute command. -u, --uri Specify an uri to execute an operation through the execute command. -v, --version Display the program version.

@@ -153,11 +155,14 @@ authFile = val

of "config", "c": if val == "": fail(115, "Configuration file not specified.") - configuration = val.parseFile + configuration = val.parseFile() configFile = val of "mount", "m": mount = true cliSettings["mount"] = %mount + of "importTags": + importTags = true + cliSettings["importTags"] = %importTags of "version", "v": echo pkgVersion quit(0)

@@ -187,6 +192,7 @@ LS.middleware = middleware

LS.authFile = authFile LS.config = configuration LS.configFile = configFile + LS.importTags = importTags LS.mount = mount LS.execution.file = exFile LS.execution.body = exBody
M src/litestorepkg/lib/core.nimsrc/litestorepkg/lib/core.nim

@@ -485,7 +485,7 @@

proc countDocuments*(store: Datastore): int64 = return store.db.getRow(SQL_COUNT_DOCUMENTS)[0].parseInt -proc importFile*(store: Datastore, f: string, dir = "/", system = false) = +proc importFile*(store: Datastore, f: string, dir = "/", system = false): string = if not f.fileExists: raise newException(EFileNotFound, "File '$1' not found." % f) let ext = f.splitFile.ext

@@ -520,6 +520,20 @@ eWarn()

raise if singleOp: store.commit() + return d_id + +proc importTags*(store: Datastore, d_id: string, tags: openArray[string]) = + let singleOp = not LS_TRANSACTION + store.begin() + try: + for tag in tags: + store.db.exec(SQL_INSERT_TAG, tag, d_id) + except: + store.rollback() + eWarn() + raise + if singleOp: + store.commit() proc optimize*(store: Datastore) = try:

@@ -545,16 +559,28 @@ eWarn()

quit(203) quit(0) -proc importDir*(store: Datastore, dir: string, system = false) = +proc getTagsForFile*(f: string): seq[string] = + result = newSeq[string]() + let tags_file = f.splitFile.dir / "_tags" + if tags_file.fileExists: + for tag in tags_file.lines: + result.add(tag) + + +proc importDir*(store: Datastore, dir: string, system = false, importTags = false) = var files = newSeq[string]() if not dir.dirExists: raise newException(EDirectoryNotFound, "Directory '$1' not found." % dir) for f in dir.walkDirRec(): if f.dirExists: continue - if f.splitFile.name.startsWith("."): + let fileName = f.splitFile.name + if fileName.startsWith("."): # Ignore hidden files continue + if fileName == "_tags" and not importTags: + # Ignore tags file unless the CLI flag was set + continue files.add(f) # Import single files in batch let batchSize = 100

@@ -567,7 +593,11 @@ LOG.debug("Dropping column indexes...")

store.db.dropIndexes() for f in files: try: - store.importFile(f, dir, system) + let docId = store.importFile(f, dir, system) + if not system and importTags: + let tags = getTagsForFile(f) + if tags.len > 0: + store.importTags(docId, tags) cFiles.inc if (cFiles-1) mod batchSize == 0: cBatches.inc

@@ -734,7 +764,8 @@

if LS.execution.operation == "" and LS.operation == opExecute: fail(111, "--operation option not specified") - + if LS.importTags and LS.operation != opImport: + fail(116, "--importTags option alowed only for import operation.") proc updateConfig*(LS: LiteStore) = let rawConfig = LS.config.pretty if LS.configFile != "":
M src/litestorepkg/lib/types.nimsrc/litestorepkg/lib/types.nim

@@ -88,6 +88,7 @@ cliSettings*: JsonNode

directory*: string manageSystemData*: bool file*: string + importTags*: bool mount*: bool readonly*: bool appname*: string