all repos — hastysite @ 8cbb70edff18e18a1d7ebabde109e773c9fa4626

A high-performance static site generator.

vendor/moustachupkg/tokenizer.nim

 1
 2
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 
import sequtils
import strutils

type
  TokenType* {.pure.} = enum
    rawtext,
    escapedvariable,
    unescapedvariable,
    section,
    invertedsection,
    comment,
    partial,
    ender,
    indenter

  Token* = tuple[tokentype: TokenType, value: string]


proc left_side_empty(tmplate: string, pivotindex: int): tuple[empty: bool, newlineindex: int] =
  var ls_i = 0
  var ls_empty = false
  var i = pivotindex - 1
  while i > -1 and tmplate[i] in {' ', '\t'}: dec(i)
  if (i == -1) or (tmplate[i] == '\l'):
    ls_i = i
    ls_empty = true
  return (empty: ls_empty, newlineindex: ls_i)


iterator tokenize*(tmplate: string): Token =
  let opening = "{{"
  var pos = 0

  while pos < tmplate.len:
    let originalpos = pos
    var closing = "}}"

    # find start of tag
    var opening_index = tmplate.find(opening, start=pos)
    if opening_index == -1:
      yield (tokentype: TokenType.rawtext, value: tmplate[pos..high(tmplate)])
      break

    #Check if the left side is empty
    var left_side = left_side_empty(tmplate, opening_index)
    var ls_empty = left_side.empty
    var ls_i = left_side.newlineindex

    #Deal with text before tag
    var beforetoken = (tokentype: TokenType.rawtext, value: "")
    if opening_index > pos:
      #safe bet for now
      beforetoken.value = tmplate[pos..opening_index-1]

    pos = opening_index + opening.len

    if not (pos < tmplate.len):
      yield (tokentype: TokenType.rawtext, value: tmplate[opening_index..high(tmplate)])
      break

    #Determine TokenType
    var tt = TokenType.escapedvariable

    case tmplate[pos]
    of '!':
      tt = TokenType.comment
      pos += 1
    of '&':
      tt = TokenType.unescapedvariable
      pos += 1
    of '{':
      tt = TokenType.unescapedvariable
      pos += 1
      closing &= "}"
    of '#':
      tt = TokenType.section
      pos += 1
    of '^':
      tt = TokenType.invertedsection
      pos += 1
    of '/':
      tt = TokenType.ender
      pos += 1
    of '>':
      tt = TokenType.partial
      pos += 1
    else:
      tt = TokenType.escapedvariable

    #find end of tag
    var closingindex = tmplate.find(closing, start=pos)
    if closingindex == -1:
      if beforetoken.value != "": yield beforetoken
      yield (tokentype: TokenType.rawtext, value: tmplate[opening_index..pos-1])
      continue

    #Check if the right side is empty
    var rs_i = 0
    var rs_empty = false
    var i = 0
    if ls_empty:
      i = closingindex + closing.len
      while i < tmplate.len and tmplate[i] in {' ', '\t'}: inc(i)
      if i == tmplate.len:
        rs_i = i - 1
        rs_empty = true
      elif tmplate[i] == '\c' and (i+1 < tmplate.len) and (tmplate[i+1] == '\l'):
        rs_i = i + 1
        rs_empty = true
      elif tmplate[i] == '\l':
        rs_i = i
        rs_empty = true
      else:
        discard

    if tt in [TokenType.comment, TokenType.section,
              TokenType.invertedsection, TokenType.ender, TokenType.partial]:
      # Standalone tokens
      if rs_empty:
        if beforetoken.value != "":
          beforetoken.value = tmplate[originalpos..ls_i]
          yield beforetoken

        if tt == TokenType.partial:
          if ls_i+1 <= opening_index-1:
            yield (tokentype: TokenType.indenter, value: tmplate[ls_i+1..opening_index-1])

        yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip)
        pos = rs_i + 1 # remove new line of this line
      else:
        if beforetoken.value != "": yield beforetoken
        yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip)
        pos = closingindex + closing.len
    else:
      if beforetoken.value != "": yield beforetoken
      yield (tokentype: tt, value: tmplate[pos..closingindex-1].strip)
      pos = closingindex + closing.len