Merge pull request #64 from tomidery/tag-documents-on-import
Fabio Cevasco h3rald@h3rald.com
Tue, 06 Apr 2021 16:27:36 +0200
5 files changed,
70 insertions(+),
7 deletions(-)
M
src/admin/md/usage.md
→
src/admin/md/usage.md
@@ -30,6 +30,7 @@ * **-p**, **-\-port** —Specify server port number (default: 9500).
* **-r**, **-\-readonly** — Allow only data retrieval operations. * **-s**, **-\-store** — Specify a datastore file (default: data.db) * **--system** — Set the system flag for import, export, and delete operations +* **--importTags** — During import read tags from '_tags' file and apply them to imported documents from the same directory. * **-t**, **--type** — Specify a content type for the body an operation to be executed via the execute command. * **-u**, **--uri** — Specify an uri to execute an operation through the execute command. * **-v**, **-\-version** — Display the program version.@@ -78,6 +79,30 @@
Import all documents stored in a directory called **system** as system documents: [litestore import -d:system --system](class:cmd) + +Import all documents stored in a directory called **media** (including subdirectories): +``` ++ media + + cars + | + _tags + | + Lamborgini.jpg + | + VW.jpg + | ` BMW.jpg + + planes + | + _tags + | + 767.jpg + | + F-16.jpg + | ` B-1.jpg + ` trains + + TGV.jpg + ` Eurostar.jpg +``` + +[litestore import -d:media --importTags](class:cmd) + +Every **_tags** file contains a list of tags, one per line, which are applied to all imported documents from the same directory. In the example above all cars and planes images will be tagged on import. The trains images, not as there is not **_tags** file in the **trains** directory. + +The individual **_tags** files are also imported. When the **--importTags** option is not set the **_tags** files are ignored and not imported. #### Exporting a directory
M
src/litestore.nim
→
src/litestore.nim
@@ -107,7 +107,7 @@ of opRun:
LS.serve runForever() of opImport: - LS.store.importDir(LS.directory, LS.manageSystemData) + LS.store.importDir(LS.directory, LS.manageSystemData, LS.importTags) of opExport: LS.store.exportDir(LS.directory, LS.manageSystemData) of opDelete:
M
src/litestorepkg/lib/cli.nim
→
src/litestorepkg/lib/cli.nim
@@ -18,6 +18,7 @@ directory:string = ""
readonly = false logLevel = "warn" system = false + importTags = false mount = false auth = newJNull() middleware = newStringTable()@@ -62,6 +63,7 @@ -p, --port Specify server port number (default: 9500).
-r, --readonly Allow only data retrieval operations. -s, --store Specify a datastore file (default: data.db) --system Set the system flag for import, export, and delete operations + --tags During import read tags from '_tags' file and apply them to imported documents from the same directory. -t, --type Specify a content type for the body an operation to be executed via the execute command. -u, --uri Specify an uri to execute an operation through the execute command. -v, --version Display the program version.@@ -153,11 +155,14 @@ authFile = val
of "config", "c": if val == "": fail(115, "Configuration file not specified.") - configuration = val.parseFile + configuration = val.parseFile() configFile = val of "mount", "m": mount = true cliSettings["mount"] = %mount + of "importTags": + importTags = true + cliSettings["importTags"] = %importTags of "version", "v": echo pkgVersion quit(0)@@ -187,6 +192,7 @@ LS.middleware = middleware
LS.authFile = authFile LS.config = configuration LS.configFile = configFile + LS.importTags = importTags LS.mount = mount LS.execution.file = exFile LS.execution.body = exBody
M
src/litestorepkg/lib/core.nim
→
src/litestorepkg/lib/core.nim
@@ -485,7 +485,7 @@
proc countDocuments*(store: Datastore): int64 = return store.db.getRow(SQL_COUNT_DOCUMENTS)[0].parseInt -proc importFile*(store: Datastore, f: string, dir = "/", system = false) = +proc importFile*(store: Datastore, f: string, dir = "/", system = false): string = if not f.fileExists: raise newException(EFileNotFound, "File '$1' not found." % f) let ext = f.splitFile.ext@@ -520,6 +520,20 @@ eWarn()
raise if singleOp: store.commit() + return d_id + +proc importTags*(store: Datastore, d_id: string, tags: openArray[string]) = + let singleOp = not LS_TRANSACTION + store.begin() + try: + for tag in tags: + store.db.exec(SQL_INSERT_TAG, tag, d_id) + except: + store.rollback() + eWarn() + raise + if singleOp: + store.commit() proc optimize*(store: Datastore) = try:@@ -545,16 +559,28 @@ eWarn()
quit(203) quit(0) -proc importDir*(store: Datastore, dir: string, system = false) = +proc getTagsForFile*(f: string): seq[string] = + result = newSeq[string]() + let tags_file = f.splitFile.dir / "_tags" + if tags_file.fileExists: + for tag in tags_file.lines: + result.add(tag) + + +proc importDir*(store: Datastore, dir: string, system = false, importTags = false) = var files = newSeq[string]() if not dir.dirExists: raise newException(EDirectoryNotFound, "Directory '$1' not found." % dir) for f in dir.walkDirRec(): if f.dirExists: continue - if f.splitFile.name.startsWith("."): + let fileName = f.splitFile.name + if fileName.startsWith("."): # Ignore hidden files continue + if fileName == "_tags" and not importTags: + # Ignore tags file unless the CLI flag was set + continue files.add(f) # Import single files in batch let batchSize = 100@@ -567,7 +593,11 @@ LOG.debug("Dropping column indexes...")
store.db.dropIndexes() for f in files: try: - store.importFile(f, dir, system) + let docId = store.importFile(f, dir, system) + if not system and importTags: + let tags = getTagsForFile(f) + if tags.len > 0: + store.importTags(docId, tags) cFiles.inc if (cFiles-1) mod batchSize == 0: cBatches.inc@@ -734,7 +764,8 @@
if LS.execution.operation == "" and LS.operation == opExecute: fail(111, "--operation option not specified") - + if LS.importTags and LS.operation != opImport: + fail(116, "--importTags option alowed only for import operation.") proc updateConfig*(LS: LiteStore) = let rawConfig = LS.config.pretty if LS.configFile != "":
M
src/litestorepkg/lib/types.nim
→
src/litestorepkg/lib/types.nim
@@ -88,6 +88,7 @@ cliSettings*: JsonNode
directory*: string manageSystemData*: bool file*: string + importTags*: bool mount*: bool readonly*: bool appname*: string