all repos — litestore @ 3248b9199405c04cc829e99803a248191ae4534b

A minimalist nosql document store.

Merge pull request #67 from tomidery/ingore-files-on-import

Fabio Cevasco h3rald@h3rald.com
Mon, 21 Jun 2021 21:11:49 +0200
commit

3248b9199405c04cc829e99803a248191ae4534b

parent

528bf55a8334db1dbcb6d2f90b900f02e40ebc02

M src/admin/md/usage.mdsrc/admin/md/usage.md

@@ -31,6 +31,7 @@ * **-r**, **-\-readonly** — Allow only data retrieval operations.

* **-s**, **-\-store** — Specify a datastore file (default: data.db) * **-\-system** — Set the system flag for import, export, and delete operations * **-\-import-tags** — During import read tags from '_tags' file and apply them to imported documents from the same directory. +* **-\-not-searchable** — During import exclude content of files and folders starting with '_' from full text search. * **-t**, **-\-type** — Specify a content type for the body an operation to be executed via the execute command. * **-u**, **-\-uri** — Specify an uri to execute an operation through the execute command. * **-v**, **-\-version** — Display the program version.

@@ -80,6 +81,8 @@ Import all documents stored in a directory called **system** as system documents:

[litestore import -d:system -\-system](class:cmd) +#### Importing a directory and tagging files + Import all documents stored in a directory called **media** (including subdirectories): ```

@@ -104,6 +107,38 @@

Every **_tags** file contains a list of tags, one per line, which are applied to all imported documents from the same directory. In the example above all cars and planes images will be tagged on import. The trains images, not as there is not **_tags** file in the **trains** directory. The individual **_tags** files are also imported. When the **\-\-import\-tags** option is not set the **_tags** files are ignored and not imported. + +#### Excluding files from full text search + +Import all documents stored in a directory called **media** (including subdirectories): + +``` ++ media + + _css + | + style1.css + | ` style2.css + + cars + | + _index.html + | + Lamborghini.md + | + VW.md + | ` BMW.md + + planes + | + _index.html + | + 767.md + | + F-16.md + | ` B-1.md + ` trains + + _index.html + * _script.js + + TGV.md + ` Eurostar.md +``` + +[litestore import -d:media -\-not-searchable](class:cmd) + +All documents are imported but the files starting which name starts with **underscore** and files inside a folder which name starts with **underscore** are excluded from full text serach. The idea is that these files have special meaning for the application: +* they should be accessible via regular URLs (unlike **system** files) +* but they content should not be searchable. #### Exporting a directory
M src/litestore.nimsrc/litestore.nim

@@ -107,7 +107,7 @@ of opRun:

LS.serve runForever() of opImport: - LS.store.importDir(LS.directory, LS.manageSystemData, LS.importTags) + LS.store.importDir(LS.directory, LS.manageSystemData, LS.importTags, LS.notSerachable) of opExport: LS.store.exportDir(LS.directory, LS.manageSystemData) of opDelete:
M src/litestorepkg/lib/cli.nimsrc/litestorepkg/lib/cli.nim

@@ -19,6 +19,7 @@ readonly = false

logLevel = "warn" system = false importTags = false + notSerachable = false mount = false auth = newJNull() middleware = newStringTable()

@@ -63,7 +64,8 @@ -p, --port Specify server port number (default: 9500).

-r, --readonly Allow only data retrieval operations. -s, --store Specify a datastore file (default: data.db) --system Set the system flag for import, export, and delete operations - --tags During import read tags from '_tags' file and apply them to imported documents from the same directory. + --import-tags During import read tags from '_tags' file and apply them to imported documents from the same directory. + --not-searchable During import exclude content of files and folders starting with '_' from full text search. -t, --type Specify a content type for the body an operation to be executed via the execute command. -u, --uri Specify an uri to execute an operation through the execute command. -v, --version Display the program version.

@@ -163,6 +165,9 @@ cliSettings["mount"] = %mount

of "import-tags": importTags = true cliSettings["import-tags"] = %importTags + of "not-searchable": + notSerachable = true + cliSettings["not-searchable"] = %notSerachable of "version", "v": echo pkgVersion quit(0)

@@ -193,6 +198,7 @@ LS.authFile = authFile

LS.config = configuration LS.configFile = configFile LS.importTags = importTags + LS.notSerachable = notSerachable LS.mount = mount LS.execution.file = exFile LS.execution.body = exBody
M src/litestorepkg/lib/core.nimsrc/litestorepkg/lib/core.nim

@@ -1,13 +1,13 @@

import x_sqlite3, x_db_sqlite as db, - strutils, os, oids, json, pegs, strtabs, strutils, + sequtils, base64, math import

@@ -302,7 +302,7 @@ var res = store.db.insertID(SQL_INSERT_DOCUMENT, id, data, contenttype,

binary, searchable, time, time) if res > 0: store.db.exec(SQL_INCREMENT_DOCS) - if binary <= 0 and searchable >= 0: + if binary <= 0 and searchable > 0: # Add to search index store.db.exec(SQL_INSERT_SEARCHCONTENT, res, id, data.toPlainText) store.addDocumentSystemTags(id, contenttype)

@@ -406,7 +406,7 @@ store.begin()

var res = store.db.execAffectedRows(SQL_UPDATE_DOCUMENT, data, contenttype, binary, searchable, currentTime(), id) if res > 0: - if binary <= 0 and searchable >= 0: + if binary <= 0 and searchable > 0: store.db.exec(SQL_UPDATE_SEARCHCONTENT, data.toPlainText, id) store.destroyDocumentSystemTags(id) store.addDocumentSystemTags(id, contenttype)

@@ -496,10 +496,10 @@

proc countDocuments*(store: Datastore): int64 = return store.db.getRow(SQL_COUNT_DOCUMENTS)[0].parseInt -proc importFile*(store: Datastore, f: string, dir = "/", system = false): string = +proc importFile*(store: Datastore, f: string, dir = "/", system = false, notSearchable = false): string = if not f.fileExists: raise newException(EFileNotFound, "File '$1' not found." % f) - let ext = f.splitFile.ext + let split = f.splitFile var d_id: string if system: # Do not save original directory name

@@ -508,10 +508,15 @@ else:

d_id = f.replace("\\", "/"); var d_contents = f.readFile var d_ct = "application/octet-stream" - if CONTENT_TYPES.hasKey(ext): - d_ct = CONTENT_TYPES[ext].replace("\"", "") + if CONTENT_TYPES.hasKey(split.ext): + d_ct = CONTENT_TYPES[split.ext].replace("\"", "") var d_binary = 0 var d_searchable = 1 + if notSearchable and (split.name.startsWith("_") or split.dir.startsWith("_")): + # Don't search in special files (and files in special folders) + # when the flag was set + d_searchable = 0 + LOG.debug("Importing not-searchable file $1", f) if d_ct.isBinary: d_binary = 1 d_searchable = 0

@@ -578,17 +583,18 @@ for tag in tags_file.lines:

result.add(tag) -proc importDir*(store: Datastore, dir: string, system = false, importTags = false) = +proc importDir*(store: Datastore, dir: string, system = false, importTags = false, notSearchable = false) = var files = newSeq[string]() if not dir.dirExists: raise newException(EDirectoryNotFound, "Directory '$1' not found." % dir) for f in dir.walkDirRec(): if f.dirExists: continue - let fileName = f.splitFile.name - if fileName.startsWith("."): - # Ignore hidden files - continue + let dirs = f.split(DirSep) + if dirs.any(proc (s: string): bool = return s.startsWith(".")): + # Ignore hidden directories and files + continue + let fileName = f.splitFile.name if fileName == "_tags" and not importTags: # Ignore tags file unless the CLI flag was set continue

@@ -604,7 +610,7 @@ LOG.debug("Dropping column indexes...")

store.db.dropIndexes() for f in files: try: - let docId = store.importFile(f, dir, system) + let docId = store.importFile(f, dir, system, notSearchable) if not system and importTags: let tags = getTagsForFile(f) if tags.len > 0:

@@ -777,6 +783,9 @@ fail(111, "--operation option not specified")

if LS.importTags and LS.operation != opImport: fail(116, "--import-tags option alowed only for import operation.") + if LS.notSerachable and LS.operation != opImport: + fail(116, "--not-searchable option alowed only for import operation.") + proc updateConfig*(LS: LiteStore) = let rawConfig = LS.config.pretty if LS.configFile != "":
M src/litestorepkg/lib/types.nimsrc/litestorepkg/lib/types.nim

@@ -89,6 +89,7 @@ directory*: string

manageSystemData*: bool file*: string importTags*: bool + notSerachable*: bool mount*: bool readonly*: bool appname*: string