@@ -31,6 +31,7 @@ * **-r**, **-\-readonly** — Allow only data retrieval operations.
 * **-s**, **-\-store** — Specify a datastore file (default: data.db)
 * **-\-system**  — Set the system flag for import, export, and delete operations
 * **-\-import-tags**  — During import read tags from '_tags' file and apply them to imported documents from the same directory.
+* **-\-not-searchable**  — During import exclude content of files and folders starting with '_' from full text search.
 * **-t**, **-\-type** — Specify a content type for the body an operation to be executed via the execute command.
 * **-u**, **-\-uri** — Specify an uri to execute an operation through the execute command.
 * **-v**, **-\-version** — Display the program version.
@@ -80,6 +81,8 @@ Import all documents stored in a directory called **system** as system documents:
 
 [litestore import -d:system -\-system](class:cmd)
 
+#### Importing a directory and tagging files
+
 Import all documents stored in a directory called **media** (including subdirectories):
 
 ```
@@ -104,6 +107,38 @@ 
 Every **_tags** file contains a list of tags, one per line, which are applied to all imported documents from the same directory. In the example above all cars and planes images will be tagged on import. The trains images, not as there is not **_tags** file in the **trains** directory.
 
 The individual **_tags** files are also imported. When the **\-\-import\-tags** option is not set the **_tags** files are ignored and not imported.
+
+#### Excluding files from full text search
+
+Import all documents stored in a directory called **media** (including subdirectories):
+
+```
++ media
+  + _css
+  | + style1.css
+  | ` style2.css
+  + cars
+  | + _index.html
+  | + Lamborghini.md
+  | + VW.md
+  | ` BMW.md
+  + planes
+  | + _index.html
+  | + 767.md
+  | + F-16.md
+  | ` B-1.md
+  ` trains
+    + _index.html
+    * _script.js
+    + TGV.md
+    ` Eurostar.md
+```   
+
+[litestore import -d:media -\-not-searchable](class:cmd)
+
+All documents are imported but the files starting which name starts with **underscore** and files inside a folder which name starts with **underscore** are excluded from full text serach. The idea is that these files have special meaning for the application:
+* they should be accessible via regular URLs (unlike **system** files)
+* but they content should not be searchable.
 
 #### Exporting a directory
 

@@ -107,7 +107,7 @@ of opRun:
         LS.serve
         runForever()
       of opImport:
-        LS.store.importDir(LS.directory, LS.manageSystemData, LS.importTags)
+        LS.store.importDir(LS.directory, LS.manageSystemData, LS.importTags, LS.notSerachable)
       of opExport:
         LS.store.exportDir(LS.directory, LS.manageSystemData)
       of opDelete:

@@ -19,6 +19,7 @@ readonly = false
   logLevel = "warn"
   system = false
   importTags = false
+  notSerachable = false
   mount = false
   auth = newJNull()
   middleware = newStringTable()
@@ -63,7 +64,8 @@ -p, --port          Specify server port number (default: 9500).
     -r, --readonly      Allow only data retrieval operations.
     -s, --store         Specify a datastore file (default: data.db)
     --system            Set the system flag for import, export, and delete operations
-    --tags              During import read tags from '_tags' file and apply them to imported documents from the same directory.
+    --import-tags       During import read tags from '_tags' file and apply them to imported documents from the same directory.
+    --not-searchable    During import exclude content of files and folders starting with '_' from full text search.
     -t, --type          Specify a content type for the body an operation to be executed via the execute command.
     -u, --uri           Specify an uri to execute an operation through the execute command.
     -v, --version       Display the program version.
@@ -163,6 +165,9 @@ cliSettings["mount"] = %mount
           of "import-tags":
             importTags = true
             cliSettings["import-tags"] = %importTags
+          of "not-searchable":
+            notSerachable = true
+            cliSettings["not-searchable"] = %notSerachable
           of "version", "v":
             echo pkgVersion
             quit(0)
@@ -193,6 +198,7 @@ LS.authFile = authFile
   LS.config = configuration
   LS.configFile = configFile
   LS.importTags = importTags
+  LS.notSerachable = notSerachable
   LS.mount = mount
   LS.execution.file = exFile
   LS.execution.body = exBody

@@ -1,13 +1,13 @@ 
 import
   x_sqlite3,
   x_db_sqlite as db,
-  strutils,
   os,
   oids,
   json,
   pegs,
   strtabs,
   strutils,
+  sequtils,
   base64,
   math
 import
@@ -302,7 +302,7 @@ var res = store.db.insertID(SQL_INSERT_DOCUMENT, id, data, contenttype,
         binary, searchable, time, time)
     if res > 0:
       store.db.exec(SQL_INCREMENT_DOCS)
-      if binary <= 0 and searchable >= 0:
+      if binary <= 0 and searchable > 0:
         # Add to search index
         store.db.exec(SQL_INSERT_SEARCHCONTENT, res, id, data.toPlainText)
       store.addDocumentSystemTags(id, contenttype)
@@ -406,7 +406,7 @@ store.begin()
     var res = store.db.execAffectedRows(SQL_UPDATE_DOCUMENT, data, contenttype,
         binary, searchable, currentTime(), id)
     if res > 0:
-      if binary <= 0 and searchable >= 0:
+      if binary <= 0 and searchable > 0:
         store.db.exec(SQL_UPDATE_SEARCHCONTENT, data.toPlainText, id)
       store.destroyDocumentSystemTags(id)
       store.addDocumentSystemTags(id, contenttype)
@@ -496,10 +496,10 @@ 
 proc countDocuments*(store: Datastore): int64 =
   return store.db.getRow(SQL_COUNT_DOCUMENTS)[0].parseInt
 
-proc importFile*(store: Datastore, f: string, dir = "/", system = false): string  =
+proc importFile*(store: Datastore, f: string, dir = "/", system = false, notSearchable = false): string  =
   if not f.fileExists:
     raise newException(EFileNotFound, "File '$1' not found." % f)
-  let ext = f.splitFile.ext
+  let split = f.splitFile
   var d_id: string
   if system:
     # Do not save original directory name
@@ -508,10 +508,15 @@ else:
     d_id = f.replace("\\", "/");
   var d_contents = f.readFile
   var d_ct = "application/octet-stream"
-  if CONTENT_TYPES.hasKey(ext):
-    d_ct = CONTENT_TYPES[ext].replace("\"", "")
+  if CONTENT_TYPES.hasKey(split.ext):
+    d_ct = CONTENT_TYPES[split.ext].replace("\"", "")
   var d_binary = 0
   var d_searchable = 1
+  if notSearchable and (split.name.startsWith("_") or split.dir.startsWith("_")):
+    # Don't search in special files (and files in special folders)
+    # when the flag was set
+    d_searchable = 0
+    LOG.debug("Importing not-searchable file $1", f)
   if d_ct.isBinary:
     d_binary = 1
     d_searchable = 0
@@ -578,17 +583,18 @@ for tag in tags_file.lines:
       result.add(tag) 
 
 
-proc importDir*(store: Datastore, dir: string, system = false, importTags = false) =
+proc importDir*(store: Datastore, dir: string, system = false, importTags = false, notSearchable = false) =
   var files = newSeq[string]()
   if not dir.dirExists:
     raise newException(EDirectoryNotFound, "Directory '$1' not found." % dir)
   for f in dir.walkDirRec():
     if f.dirExists:
       continue
-    let fileName = f.splitFile.name
-    if fileName.startsWith("."):
-      # Ignore hidden files
-      continue
+    let dirs = f.split(DirSep)  
+    if dirs.any(proc (s: string): bool = return s.startsWith(".")):
+      # Ignore hidden directories and files
+      continue  
+    let fileName = f.splitFile.name    
     if fileName == "_tags" and not importTags:
       # Ignore tags file unless the CLI flag was set
       continue
@@ -604,7 +610,7 @@ LOG.debug("Dropping column indexes...")
   store.db.dropIndexes()
   for f in files:
     try:
-      let docId = store.importFile(f, dir, system)
+      let docId = store.importFile(f, dir, system, notSearchable)
       if not system and importTags:
         let tags = getTagsForFile(f)
         if tags.len > 0:
@@ -777,6 +783,9 @@ fail(111, "--operation option not specified")
 
   if LS.importTags and LS.operation != opImport:
     fail(116, "--import-tags option alowed only for import operation.")
+  if LS.notSerachable and LS.operation != opImport:
+    fail(116, "--not-searchable option alowed only for import operation.")
+
 proc updateConfig*(LS: LiteStore) =
   let rawConfig = LS.config.pretty
   if LS.configFile != "":

@@ -89,6 +89,7 @@ directory*: string
     manageSystemData*: bool
     file*: string
     importTags*: bool
+    notSerachable*: bool
     mount*: bool
     readonly*: bool
     appname*: string