all repos — hastysite @ f695a875ef9608090f1b867442e3192e8481e807

A high-performance static site generator.

Now using Nim CFG instead of YAML for metadata.
h3rald h3rald@h3rald.com
Sat, 10 Jun 2017 18:45:46 +0200
commit

f695a875ef9608090f1b867442e3192e8481e807

parent

aa54f112c8c711243e3a962099f3d2f0944eb88a

6 files changed, 133 insertions(+), 681 deletions(-)

jump to
M hastysite.nimhastysite.nim

@@ -1,22 +1,21 @@

import json, strutils, - pegs, os, securehash, sequtils, tables, critbits, streams, + parsecfg, logging -import - packages/NimYAML/yaml, +import packages/min/min, - packages/hastyscribe/hastyscribe + packages/hastyscribe/hastyscribe, + packages/moustachu/src/moustachu import - vendor/moustachu, config type

@@ -47,103 +46,103 @@

#### min Library proc hastysite_module*(i: In, hs: HastySite) = - i.define() + let def = i.define() - .symbol("metadata") do (i: In): - i.push i.fromJson(hs.metadata) + def.symbol("metadata") do (i: In): + i.push i.fromJson(hs.metadata) - .symbol("settings") do (i: In): - i.push i.fromJson(hs.settings) + def.symbol("settings") do (i: In): + i.push i.fromJson(hs.settings) - .symbol("modified") do (i: In): - var modified = newSeq[MinValue](0) - for j in hs.files.modified: - modified.add i.fromJson(j) - i.push modified.newVal(i.scope) + def.symbol("modified") do (i: In): + var modified = newSeq[MinValue](0) + for j in hs.files.modified: + modified.add i.fromJson(j) + i.push modified.newVal(i.scope) - .symbol("output") do (i: In): - i.push hs.dirs.output.newVal + def.symbol("output") do (i: In): + i.push hs.dirs.output.newVal - .symbol("input-fread") do (i: In): - var d: MinValue - i.reqDictionary d - let t = d.dget("type".newVal).getString - let path = d.dget("path".newVal).getString - var contents = "" - if t == "content": - contents = readFile(hs.dirs.tempContents/path) - else: - contents = readFile(hs.dirs.assets/path) - i.push contents.newVal + def.symbol("input-fread") do (i: In): + var d: MinValue + i.reqDictionary d + let t = d.dget("type".newVal).getString + let path = d.dget("path".newVal).getString + var contents = "" + if t == "content": + contents = readFile(hs.dirs.tempContents/path) + else: + contents = readFile(hs.dirs.assets/path) + i.push contents.newVal - .symbol("output-fwrite") do (i: In): - var d: MinValue - i.reqDictionary d - let id = d.dget("id".newVal).getString - let ext = d.dget("ext".newVal).getString - var contents = "" - try: - contents = d.dget("contents".newVal).getString - except: - raise MetadataRequiredException(msg: "Metadata key 'contents' not found in dictionary.") - let outfile = hs.dirs.output/id&ext - outfile.parentDir.createDir - writeFile(outfile, contents) + def.symbol("output-fwrite") do (i: In): + var d: MinValue + i.reqDictionary d + let id = d.dget("id".newVal).getString + let ext = d.dget("ext".newVal).getString + var contents = "" + try: + contents = d.dget("contents".newVal).getString + except: + raise MetadataRequiredException(msg: "Metadata key 'contents' not found in dictionary.") + let outfile = hs.dirs.output/id&ext + outfile.parentDir.createDir + writeFile(outfile, contents) - .symbol("copy2output") do (i: In): - var d: MinValue - i.reqDictionary d - let t = d.dget("type".newVal).getString - let path = d.dget("path".newVal).getString - var infile, outfile: string - if t == "content": - infile = hs.dirs.tempContents/path - outfile = hs.dirs.output/path - else: - infile = hs.dirs.assets/path - outfile = hs.dirs.output/path - notice " - Copying: ", infile, " -> ", outfile - outfile.parentDir.createDir - copyFileWithPermissions(infile, outfile) + def.symbol("copy2output") do (i: In): + var d: MinValue + i.reqDictionary d + let t = d.dget("type".newVal).getString + let path = d.dget("path".newVal).getString + var infile, outfile: string + if t == "content": + infile = hs.dirs.tempContents/path + outfile = hs.dirs.output/path + else: + infile = hs.dirs.assets/path + outfile = hs.dirs.output/path + notice " - Copying: ", infile, " -> ", outfile + outfile.parentDir.createDir + copyFileWithPermissions(infile, outfile) - .symbol("content?") do (i: In): - var d: MinValue - i.reqDictionary d - let t = d.dget("type".newVal).getString - let r = t == "content" - i.push r.newVal + def.symbol("content?") do (i: In): + var d: MinValue + i.reqDictionary d + let t = d.dget("type".newVal).getString + let r = t == "content" + i.push r.newVal - .symbol("asset?") do (i: In): - var d: MinValue - i.reqDictionary d - let t = d.dget("type".newVal).getString - let r = t == "asset" - i.push r.newVal + def.symbol("asset?") do (i: In): + var d: MinValue + i.reqDictionary d + let t = d.dget("type".newVal).getString + let r = t == "asset" + i.push r.newVal - .symbol("mustache") do (i: In): - var t, c: MinValue - i.reqQuotationAndString c, t - if not c.isDictionary: - raise DictionaryRequiredException(msg: "No dictionary provided as template context.") - let ctx = newContext(%c) - let tplname = t.getString & ".mustache" - let tpl = readFile(hs.dirs.templates/tplname) - i.push tpl.render(ctx, hs.dirs.templates).newval + def.symbol("mustache") do (i: In): + var t, c: MinValue + i.reqQuotationAndString c, t + if not c.isDictionary: + raise DictionaryRequiredException(msg: "No dictionary provided as template context.") + let ctx = newContext(%c) + let tplname = t.getString & ".mustache" + let tpl = readFile(hs.dirs.templates/tplname) + i.push tpl.render(ctx, hs.dirs.templates).newval - .symbol("markdown") do (i: In): - var t, c: MinValue - i.reqQuotationAndString c, t - if not c.isDictionary: - raise DictionaryRequiredException(msg: "No dictionary provided for markdown processor fields.") - let options = HastyOptions(toc: false, output: nil, css: nil, watermark: nil, fragment: true) - var fields = initTable[string, proc():string]() - for item in c.qVal: - fields[item.qVal[0].getString] = proc(): string = return $$item.qVal[1] - var hastyscribe = newHastyScribe(options, fields) - let file = t.getString() - i.push hastyscribe.compileFragment(file, hs.dirs.contents).newVal + def.symbol("markdown") do (i: In): + var t, c: MinValue + i.reqQuotationAndString c, t + if not c.isDictionary: + raise DictionaryRequiredException(msg: "No dictionary provided for markdown processor fields.") + let options = HastyOptions(toc: false, output: nil, css: nil, watermark: nil, fragment: true) + var fields = initTable[string, proc():string]() + for item in c.qVal: + fields[item.qVal[0].getString] = proc(): string = return $$item.qVal[1] + var hastyscribe = newHastyScribe(options, fields) + let file = t.getString() + i.push hastyscribe.compileFragment(file, hs.dirs.contents).newVal - .finalize("hastysite") + def.finalize("hastysite") #### Helper Functions

@@ -151,7 +150,7 @@ proc preprocessContent(file, dir: string, obj: var JsonNode): string =

let fileid = file.replace(dir, "") var f: File discard f.open(file) - var s, yaml = "" + var s, cfg = "" result = "" var delimiter = 0 try:

@@ -159,22 +158,29 @@ while f.readLine(s):

if delimiter >= 2: result &= s&"\n" else: - if s.match(peg"'-' '-' '-' '-'*"): + if s.startsWith("----"): delimiter.inc else: - yaml &= s&"\n" + cfg &= s&"\n" except: discard if not obj.hasKey("contents"): obj["contents"] = newJObject() var meta = newJObject(); if delimiter < 2: - result = yaml + result = cfg else: try: - let docs = yaml.loadToJson() - if docs.len > 0: - meta = docs[0] + let ss = newStringStream(cfg) + var p: CfgParser + p.open(ss, file) + while true: + var e = next(p) + case e.kind + of cfgKeyValuePair: + meta[e.key] = newJString(e.value) + else: + discard except: meta = newJObject() meta["path"] = %fileid
M hastysite.nimblehastysite.nimble

@@ -7,4 +7,4 @@ license = "MIT"

bin = "hastysite" [Deps] -requires: "nim >= 0.15.0, yaml >= 0.7.0, hastyscribe >= 1.4.0, minim >= 0.3.0" +requires: "nim >= 0.17.0
M nifty.jsonnifty.json

@@ -1,53 +1,41 @@

{ - "storage": "packages", - "commands": - { - "install": - { - "git+src": - { + "storage": "packages", + "commands": { + "install": { + "git+src": { "cmd": "git clone {{src}} --depth 1" - }, - "git+src+tag": - { + }, + "git+src+tag": { "cmd": "git clone --branch {{tag}} {{src}} --depth 1" } - }, - "update": - { - "git+name": - { - "cmd": "git pull", + }, + "update": { + "git+name": { + "cmd": "git pull", "pwd": "{{name}}" } } - }, - "packages": - { - "yaml": - { - "name": "yaml", - "git": true, - "src": "https://github.com/flyx/NimYAML.git", - "tag": "v0.8.0" - }, - "hastyscribe": - { - "name": "hastyscribe", - "git": true, + }, + "packages": { + "hastyscribe": { + "name": "hastyscribe", + "git": true, "src": "https://github.com/h3rald/hastyscribe.git" - }, - "min": - { - "name": "min", - "git": true, + }, + "min": { + "name": "min", + "git": true, "src": "https://github.com/h3rald/min.git" - }, - "commandeer": - { - "name": "commandeer", - "src": "https://github.com/fenekku/commandeer.git", + }, + "commandeer": { + "name": "commandeer", + "src": "https://github.com/fenekku/commandeer.git", + "git": true + }, + "moustachu": { + "name": "moustachu", + "src": "https://github.com/h3rald/moustachu", "git": true } } -} +}
D vendor/moustachu.nim

@@ -1,226 +0,0 @@

- -## A mustache templating engine written in Nim. - -import strutils -import sequtils -import os - -import moustachupkg/context -import moustachupkg/tokenizer - -export context - -let - htmlReplaceBy = [("&", "&amp;"), - ("<", "&lt;"), - (">", "&gt;"), - ("\\", "&#92;"), - ("\"", "&quot;")] - - -proc lookupContext(contextStack: seq[Context], tagkey: string): Context = - ## Return the Context associated with `tagkey` where `tagkey` - ## can be a dotted tag e.g. a.b.c - ## If the Context at `tagkey` does not exist, return nil. - var currCtx = contextStack[contextStack.high] - if tagkey == ".": return currCtx - let subtagkeys = tagkey.split(".") - for i in countDown(contextStack.high, contextStack.low): - currCtx = contextStack[i] - - for subtagkey in subtagkeys: - currCtx = currCtx[subtagkey] - if currCtx == nil: - break - - if currCtx != nil: - return currCtx - - return currCtx - -proc lookupString(contextStack: seq[Context], tagkey: string): string = - ## Return the string associated with `tagkey` in Context `c`. - ## If the Context at `tagkey` does not exist, return the empty string. - result = lookupContext(contextStack, tagkey).toString() - -proc ignore(tag: string, tokens: seq[Token], index: int): int = - #ignore - var i = index + 1 - var nexttoken = tokens[i] - var openedsections = 1 - let lentokens = len(tokens) - - while i < lentokens and openedsections > 0: - if nexttoken.value == tag: - if nexttoken.tokentype in [TokenType.section, TokenType.invertedsection]: - openedsections += 1 - elif nexttoken.tokentype == TokenType.ender: - openedsections -= 1 - else: discard - else: discard - - i += 1 - nexttoken = tokens[i] - - return i - -proc parallelReplace(str: string, - substitutions: openArray[tuple[pattern: string, by: string]]): string = - ## Returns a modified copy of `str` with the `substitutions` applied - result = str - for sub in substitutions: - result = result.replace(sub[0], sub[1]) - -proc render(tmplate: string, contextStack: seq[Context], pwd="."): string = - ## Take a mustache template `tmplate` and an evaluation Context `c` - ## and return the rendered string. This is the main procedure. - var renderings : seq[string] = @[] - - #Object - var sections : seq[string] = @[] - var contextStack = contextStack - - #Array - var loopStartPositions : seq[int] = @[] - var loopCounters : seq[int] = @[] - - #Indentation - var indentation = "" - - let tokens = toSeq(tokenizer.tokenize(tmplate)) - let lentokens = len(tokens) - - var index = 0 - - while index < lentokens: - let token = tokens[index] - - case token.tokentype - of TokenType.comment: - discard - - of TokenType.escapedvariable: - var viewvalue = contextStack.lookupString(token.value) - viewvalue = viewvalue.parallelReplace(htmlReplaceBy) - renderings.add(viewvalue) - - of TokenType.unescapedvariable: - var viewvalue = contextStack.lookupString(token.value) - renderings.add(viewvalue) - - of TokenType.section: - let ctx = contextStack.lookupContext(token.value) - if ctx == nil: - index = ignore(token.value, tokens, index) - continue - elif ctx.kind == CObject: - # enter a new section - if ctx.len == 0: - index = ignore(token.value, tokens, index) - continue - else: - contextStack.add(ctx) - sections.add(token.value) - elif ctx.kind == CArray: - # update the array loop stacks - if ctx.len == 0: - index = ignore(token.value, tokens, index) - continue - else: - #do looping - index += 1 - loopStartPositions.add(index) - loopCounters.add(ctx.len) - sections.add(token.value) - contextStack.add(ctx[ctx.len - loopCounters[^1]]) - continue - elif ctx.kind == CValue: - if not ctx: - index = ignore(token.value, tokens, index) - continue - else: discard #we will render the text inside the section - - of TokenType.invertedsection: - let ctx = contextStack.lookupContext(token.value) - if ctx != nil: - if ctx.kind == CObject: - index = ignore(token.value, tokens, index) - continue - elif ctx.kind == CArray: - if ctx.len != 0: - index = ignore(token.value, tokens, index) - continue - elif ctx.kind == CValue: - if ctx: - index = ignore(token.value, tokens, index) - continue - else: discard #we will render the text inside the section - - of TokenType.ender: - var ctx = contextStack.lookupContext(token.value) - if ctx != nil: - if ctx.kind == CObject: - discard contextStack.pop() - discard sections.pop() - elif ctx.kind == CArray: - if ctx.len > 0: - loopCounters[^1] -= 1 - discard contextStack.pop() - if loopCounters[^1] == 0: - discard loopCounters.pop() - discard loopStartPositions.pop() - discard sections.pop() - else: - index = loopStartPositions[^1] - contextStack.add(ctx[ctx.len - loopCounters[^1]]) - continue - - of TokenType.indenter: - if token.value != "": - indentation = token.value - renderings.add(indentation) - - of TokenType.partial: - var partialTemplate = pwd.joinPath(token.value & ".mustache").readFile - partialTemplate = partialTemplate.replace("\n", "\n" & indentation) - if indentation != "": - partialTemplate = partialTemplate.strip(leading=false, chars={' '}) - indentation = "" - renderings.add(render(partialTemplate, contextStack, pwd)) - - else: - renderings.add(token.value) - - index += 1 - - result = join(renderings, "") - -proc render*(tmplate: string, c: Context, pwd="."): string = - var contextStack = @[c] - result = tmplate.render(contextStack, pwd) - - -when isMainModule: - import json - import os - import commandeer - - proc usage(): string = - result = "Usage! moustachu <context>.json <template>.mustache [--file=<outputFilename>]" - - commandline: - argument jsonFilename, string - argument tmplateFilename, string - option outputFilename, string, "file", "f" - exitoption "help", "h", usage() - exitoption "version", "v", "0.10.3" - errormsg usage() - - var c = newContext(parseFile(jsonFilename)) - var tmplate = readFile(tmplateFilename) - var pwd = tmplateFilename.parentDir() - - if outputFilename.isNil(): - echo render(tmplate, c, pwd) - else: - writeFile(outputFilename, render(tmplate, c, pwd))
D vendor/moustachupkg/context.nim

@@ -1,179 +0,0 @@

-import json -import sequtils -import strutils -import tables - - -type - ContextKind* = enum ## possible Context types - CArray, - CObject, - CValue - - ## Context used to render a mustache template - Context* = ref ContextObj - ContextObj = object - case kind*: ContextKind - of CValue: - val: JsonNode - of CArray: - elems: seq[Context] - of CObject: - fields: Table[string, Context] - -## Builders - -proc newContext*(j : JsonNode = nil): Context = - ## Create a new Context based on a JsonNode object - new(result) - if j == nil: - result.kind = CObject - result.fields = initTable[string, Context](4) - else: - case j.kind - of JObject: - result.kind = CObject - result.fields = initTable[string, Context](4) - for key, val in pairs(j.fields): - result.fields[key] = newContext(val) - of JArray: - result.kind = CArray - result.elems = @[] - for val in j.elems: - result.elems.add(newContext(val)) - else: - result.kind = CValue - result.val = j - -proc newArrayContext*(): Context = - ## Create a new Context of kind CArray - new(result) - result.kind = CArray - result.elems = @[] - -## Getters - -proc `[]`*(c: Context, key: string): Context = - ## Return the Context associated with `key`. - ## If the Context at `key` does not exist, return nil. - assert(c != nil) - if c.kind != CObject: return nil - if c.fields.hasKey(key): return c.fields[key] else: return nil - -proc `[]`*(c: Context, index: int): Context = - assert(c != nil) - if c.kind != CArray: return nil else: return c.elems[index] - -## Setters - -proc `[]=`*(c: var Context, key: string, value: Context) = - ## Assign a context `value` to `key` in context `c` - assert(c.kind == CObject) - c.fields[key] = value - -proc `[]=`*(c: var Context, key: string, value: JsonNode) = - ## Convert and assign `value` to `key` in `c` - assert(c.kind == CObject) - c[key] = newContext(value) - -proc `[]=`*(c: var Context; key: string, value: BiggestInt) = - ## Assign `value` to `key` in Context `c` - assert(c.kind == CObject) - c[key] = newContext(newJInt(value)) - -proc `[]=`*(c: var Context; key: string, value: string) = - ## Assign `value` to `key` in Context `c` - assert(c.kind == CObject) - c[key] = newContext(newJString(value)) - -proc `[]=`*(c: var Context; key: string, value: float) = - ## Assign `value` to `key` in Context `c` - assert(c.kind == CObject) - c[key] = newContext(newJFloat(value)) - -proc `[]=`*(c: var Context; key: string, value: bool) = - ## Assign `value` to `key` in Context `c` - assert(c.kind == CObject) - c[key] = newContext(newJBool(value)) - -proc `[]=`*(c: var Context, key: string, value: openarray[Context]) = - ## Assign `value` to `key` in Context `c` - assert(c.kind == CObject) - var contextList = newArrayContext() - for v in value: - contextList.elems.add(v) - c[key] = contextList - -proc `[]=`*(c: var Context, key: string, value: openarray[string]) = - ## Assign `value` to `key` in Context `c` - assert(c.kind == CObject) - c[key] = map(value, proc(x: string): Context = newContext(newJString(x))) - -proc `[]=`*(c: var Context, key: string, value: openarray[int]) = - ## Assign `value` to `key` in Context `c` - assert(c.kind == CObject) - c[key] = map(value, proc(x: int): Context = newContext(newJInt(x))) - -proc `[]=`*(c: var Context, key: string, value: openarray[float]) = - ## Assign `value` to `key` in Context `c` - assert(c.kind == CObject) - c[key] = map(value, proc(x: float): Context = newContext(newJFloat(x))) - -proc `[]=`*(c: var Context, key: string, value: openarray[bool]) = - ## Assign `value` to `key` in Context `c` - assert(c.kind == CObject) - c[key] = map(value, proc(x: bool): Context = newContext(newJBool(x))) - -## Printers - -proc `$`*(c: Context): string = - ## Return a string representing the context. Useful for debugging - result = "Context->[kind: " & $c.kind - case c.kind - of CValue: result &= "\nval: " & $c.val - of CArray: - var strArray = map(c.elems, proc(c: Context): string = $c) - result &= "\nelems: [" & join(strArray, ", ") & "]" - of CObject: - var strArray : seq[string] = @[] - for key, val in pairs(c.fields): - strArray.add(key & ": " & $val) - result &= "\nfields: {" & join(strArray, ", ") & "}" - result &= "\n]" - -proc toString*(c: Context): string = - ## Return string representation of `c` relevant to mustache - if c != nil: - if c.kind == CValue: - case c.val.kind - of JString: - return c.val.str - of JFloat: - return c.val.fnum.formatFloat(ffDefault, 0) - of JInt: - return $c.val.num - of JNull: - return "" - of JBool: - return if c.val.bval: "true" else: "" - else: - return $c.val - else: - return $c - else: - return "" - -proc len*(c: Context): int = - if c.kind == CArray: - result = c.elems.len - elif c.kind == CObject: - result = c.fields.len - else: discard - -converter toBool*(c: Context): bool = - assert(c.kind == CValue) - case c.val.kind - of JBool: result = c.val.bval - of JNull: result = false - of JString: result = c.val.str != "" - else: result = true
D vendor/moustachupkg/tokenizer.nim

@@ -1,137 +0,0 @@

-import sequtils -import strutils - -type - TokenType* {.pure.} = enum - rawtext, - escapedvariable, - unescapedvariable, - section, - invertedsection, - comment, - partial, - ender, - indenter - - Token* = tuple[tokentype: TokenType, value: string] - - -proc left_side_empty(tmplate: string, pivotindex: int): tuple[empty: bool, newlineindex: int] = - var ls_i = 0 - var ls_empty = false - var i = pivotindex - 1 - while i > -1 and tmplate[i] in {' ', '\t'}: dec(i) - if (i == -1) or (tmplate[i] == '\l'): - ls_i = i - ls_empty = true - return (empty: ls_empty, newlineindex: ls_i) - - -iterator tokenize*(tmplate: string): Token = - let opening = "{{" - var pos = 0 - - while pos < tmplate.len: - let originalpos = pos - var closing = "}}" - - # find start of tag - var opening_index = tmplate.find(opening, start=pos) - if opening_index == -1: - yield (tokentype: TokenType.rawtext, value: tmplate[pos..high(tmplate)]) - break - - #Check if the left side is empty - var left_side = left_side_empty(tmplate, opening_index) - var ls_empty = left_side.empty - var ls_i = left_side.newlineindex - - #Deal with text before tag - var beforetoken = (tokentype: TokenType.rawtext, value: "") - if opening_index > pos: - #safe bet for now - beforetoken.value = tmplate[pos..opening_index-1] - - pos = opening_index + opening.len - - if not (pos < tmplate.len): - yield (tokentype: TokenType.rawtext, value: tmplate[opening_index..high(tmplate)]) - break - - #Determine TokenType - var tt = TokenType.escapedvariable - - case tmplate[pos] - of '!': - tt = TokenType.comment - pos += 1 - of '&': - tt = TokenType.unescapedvariable - pos += 1 - of '{': - tt = TokenType.unescapedvariable - pos += 1 - closing &= "}" - of '#': - tt = TokenType.section - pos += 1 - of '^': - tt = TokenType.invertedsection - pos += 1 - of '/': - tt = TokenType.ender - pos += 1 - of '>': - tt = TokenType.partial - pos += 1 - else: - tt = TokenType.escapedvariable - - #find end of tag - var closingindex = tmplate.find(closing, start=pos) - if closingindex == -1: - if beforetoken.value != "": yield beforetoken - yield (tokentype: TokenType.rawtext, value: tmplate[opening_index..pos-1]) - continue - - #Check if the right side is empty - var rs_i = 0 - var rs_empty = false - var i = 0 - if ls_empty: - i = closingindex + closing.len - while i < tmplate.len and tmplate[i] in {' ', '\t'}: inc(i) - if i == tmplate.len: - rs_i = i - 1 - rs_empty = true - elif tmplate[i] == '\c' and (i+1 < tmplate.len) and (tmplate[i+1] == '\l'): - rs_i = i + 1 - rs_empty = true - elif tmplate[i] == '\l': - rs_i = i - rs_empty = true - else: - discard - - if tt in [TokenType.comment, TokenType.section, - TokenType.invertedsection, TokenType.ender, TokenType.partial]: - # Standalone tokens - if rs_empty: - if beforetoken.value != "": - beforetoken.value = tmplate[originalpos..ls_i] - yield beforetoken - - if tt == TokenType.partial: - if ls_i+1 <= opening_index-1: - yield (tokentype: TokenType.indenter, value: tmplate[ls_i+1..opening_index-1]) - - yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip) - pos = rs_i + 1 # remove new line of this line - else: - if beforetoken.value != "": yield beforetoken - yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip) - pos = closingindex + closing.len - else: - if beforetoken.value != "": yield beforetoken - yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip) - pos = closingindex + closing.len