all repos — litestore @ cd09341dc728f9f2db00991912e0ac1d59a96507

A minimalist nosql document store.

Do not include files starting with underscore in the full text search
kaminski kaminski@naew.nato.int
Mon, 31 May 2021 16:56:48 +0200
commit

cd09341dc728f9f2db00991912e0ac1d59a96507

parent

dfbd4e4b274b772c088752447d96e2e56772369d

M src/admin/md/usage.mdsrc/admin/md/usage.md

@@ -31,6 +31,7 @@ * **-r**, **-\-readonly** — Allow only data retrieval operations.

* **-s**, **-\-store** — Specify a datastore file (default: data.db) * **-\-system** — Set the system flag for import, export, and delete operations * **-\-import-tags** — During import read tags from '_tags' file and apply them to imported documents from the same directory. +* **-\-not-searchable** — During import exclude content of files and folders starting with '_' from full text search. * **-t**, **-\-type** — Specify a content type for the body an operation to be executed via the execute command. * **-u**, **-\-uri** — Specify an uri to execute an operation through the execute command. * **-v**, **-\-version** — Display the program version.

@@ -80,6 +81,8 @@ Import all documents stored in a directory called **system** as system documents:

[litestore import -d:system -\-system](class:cmd) +#### Importing a directory and tagging files + Import all documents stored in a directory called **media** (including subdirectories): ```

@@ -104,6 +107,38 @@

Every **_tags** file contains a list of tags, one per line, which are applied to all imported documents from the same directory. In the example above all cars and planes images will be tagged on import. The trains images, not as there is not **_tags** file in the **trains** directory. The individual **_tags** files are also imported. When the **\-\-import\-tags** option is not set the **_tags** files are ignored and not imported. + +#### Excluding files from full text search + +Import all documents stored in a directory called **media** (including subdirectories): + +``` ++ media + + _css + | + style1.css + | ` style2.css + + cars + | + _index.html + | + Lamborghini.md + | + VW.md + | ` BMW.md + + planes + | + _index.html + | + 767.md + | + F-16.md + | ` B-1.md + ` trains + + _index.html + * _script.js + + TGV.md + ` Eurostar.md +``` + +[litestore import -d:media -\-not-searchable](class:cmd) + +All documents are imported but the files starting which name starts with **underscore** and files inside a folder which name starts with **underscore** are excluded from full text serach. The idea is that these files have special meaning for the application: +* they should be accessible via regular URLs (unlike **system** files) +* but they content should not be searchable. #### Exporting a directory
M src/litestore.nimsrc/litestore.nim

@@ -107,7 +107,7 @@ of opRun:

LS.serve runForever() of opImport: - LS.store.importDir(LS.directory, LS.manageSystemData, LS.importTags) + LS.store.importDir(LS.directory, LS.manageSystemData, LS.importTags, LS.notSerachable) of opExport: LS.store.exportDir(LS.directory, LS.manageSystemData) of opDelete:
M src/litestorepkg/lib/cli.nimsrc/litestorepkg/lib/cli.nim

@@ -19,6 +19,7 @@ readonly = false

logLevel = "warn" system = false importTags = false + notSerachable = false mount = false auth = newJNull() middleware = newStringTable()

@@ -63,7 +64,8 @@ -p, --port Specify server port number (default: 9500).

-r, --readonly Allow only data retrieval operations. -s, --store Specify a datastore file (default: data.db) --system Set the system flag for import, export, and delete operations - --tags During import read tags from '_tags' file and apply them to imported documents from the same directory. + --import-tags During import read tags from '_tags' file and apply them to imported documents from the same directory. + --not-searchable During import exclude content of files and folders starting with '_' from full text search. -t, --type Specify a content type for the body an operation to be executed via the execute command. -u, --uri Specify an uri to execute an operation through the execute command. -v, --version Display the program version.

@@ -163,6 +165,9 @@ cliSettings["mount"] = %mount

of "import-tags": importTags = true cliSettings["import-tags"] = %importTags + of "not-searchable": + notSerachable = true + cliSettings["not-searchable"] = %notSerachable of "version", "v": echo pkgVersion quit(0)

@@ -193,6 +198,7 @@ LS.authFile = authFile

LS.config = configuration LS.configFile = configFile LS.importTags = importTags + LS.notSerachable = notSerachable LS.mount = mount LS.execution.file = exFile LS.execution.body = exBody
M src/litestorepkg/lib/core.nimsrc/litestorepkg/lib/core.nim

@@ -302,7 +302,7 @@ var res = store.db.insertID(SQL_INSERT_DOCUMENT, id, data, contenttype,

binary, searchable, time, time) if res > 0: store.db.exec(SQL_INCREMENT_DOCS) - if binary <= 0 and searchable >= 0: + if binary <= 0 and searchable > 0: # Add to search index store.db.exec(SQL_INSERT_SEARCHCONTENT, res, id, data.toPlainText) store.addDocumentSystemTags(id, contenttype)

@@ -406,7 +406,7 @@ store.begin()

var res = store.db.execAffectedRows(SQL_UPDATE_DOCUMENT, data, contenttype, binary, searchable, currentTime(), id) if res > 0: - if binary <= 0 and searchable >= 0: + if binary <= 0 and searchable > 0: store.db.exec(SQL_UPDATE_SEARCHCONTENT, data.toPlainText, id) store.destroyDocumentSystemTags(id) store.addDocumentSystemTags(id, contenttype)

@@ -496,10 +496,10 @@

proc countDocuments*(store: Datastore): int64 = return store.db.getRow(SQL_COUNT_DOCUMENTS)[0].parseInt -proc importFile*(store: Datastore, f: string, dir = "/", system = false): string = +proc importFile*(store: Datastore, f: string, dir = "/", system = false, notSearchable = false): string = if not f.fileExists: raise newException(EFileNotFound, "File '$1' not found." % f) - let ext = f.splitFile.ext + let split = f.splitFile var d_id: string if system: # Do not save original directory name

@@ -508,10 +508,15 @@ else:

d_id = f.replace("\\", "/"); var d_contents = f.readFile var d_ct = "application/octet-stream" - if CONTENT_TYPES.hasKey(ext): - d_ct = CONTENT_TYPES[ext].replace("\"", "") + if CONTENT_TYPES.hasKey(split.ext): + d_ct = CONTENT_TYPES[split.ext].replace("\"", "") var d_binary = 0 var d_searchable = 1 + if notSearchable and (split.name.startsWith("_") or split.dir.startsWith("_")): + # Don't search in special files (and files in special folders) + # when the flag was set + d_searchable = 0 + LOG.debug("Importing not-searchable file $1", f) if d_ct.isBinary: d_binary = 1 d_searchable = 0

@@ -578,7 +583,7 @@ for tag in tags_file.lines:

result.add(tag) -proc importDir*(store: Datastore, dir: string, system = false, importTags = false) = +proc importDir*(store: Datastore, dir: string, system = false, importTags = false, notSearchable = false) = var files = newSeq[string]() if not dir.dirExists: raise newException(EDirectoryNotFound, "Directory '$1' not found." % dir)

@@ -604,7 +609,7 @@ LOG.debug("Dropping column indexes...")

store.db.dropIndexes() for f in files: try: - let docId = store.importFile(f, dir, system) + let docId = store.importFile(f, dir, system, notSearchable) if not system and importTags: let tags = getTagsForFile(f) if tags.len > 0:

@@ -777,6 +782,9 @@ fail(111, "--operation option not specified")

if LS.importTags and LS.operation != opImport: fail(116, "--import-tags option alowed only for import operation.") + if LS.notSerachable and LS.operation != opImport: + fail(116, "--not-searchable option alowed only for import operation.") + proc updateConfig*(LS: LiteStore) = let rawConfig = LS.config.pretty if LS.configFile != "":
M src/litestorepkg/lib/types.nimsrc/litestorepkg/lib/types.nim

@@ -89,6 +89,7 @@ directory*: string

manageSystemData*: bool file*: string importTags*: bool + notSerachable*: bool mount*: bool readonly*: bool appname*: string