/lisp/tokens.go

https://github.com/apeace/go-lisp · Go · 160 lines · 147 code · 12 blank · 1 comment · 36 complexity · 9dc6679ef7b464c3f9cdd5e6321f2440 MD5 · raw file

  1. package lisp
  2. import (
  3. "fmt"
  4. "regexp"
  5. "strconv"
  6. )
  7. type Tokens []*Token
  8. type tokenType uint8
  9. type Token struct {
  10. typ tokenType
  11. val string
  12. }
  13. type Pattern struct {
  14. typ tokenType
  15. regexp *regexp.Regexp
  16. }
  17. func (t Token) String() string {
  18. return fmt.Sprintf("%v", t.val)
  19. }
  20. const (
  21. whitespaceToken tokenType = iota
  22. commentToken
  23. stringToken
  24. numberToken
  25. openToken
  26. closeToken
  27. symbolToken
  28. )
  29. func patterns() []Pattern {
  30. return []Pattern{
  31. {whitespaceToken, regexp.MustCompile(`^\s+`)},
  32. {commentToken, regexp.MustCompile(`^;.*`)},
  33. {stringToken, regexp.MustCompile(`^("(\\.|[^"])*")`)},
  34. {numberToken, regexp.MustCompile(`^((([0-9]+)?\.)?[0-9]+)`)},
  35. {openToken, regexp.MustCompile(`^(\()`)},
  36. {closeToken, regexp.MustCompile(`^(\))`)},
  37. {symbolToken, regexp.MustCompile(`^('|[^\s();]+)`)},
  38. }
  39. }
  40. func NewTokens(program string) (tokens Tokens) {
  41. for pos := 0; pos < len(program); {
  42. for _, pattern := range patterns() {
  43. if matches := pattern.regexp.FindStringSubmatch(program[pos:]); matches != nil {
  44. if len(matches) > 1 {
  45. tokens = append(tokens, &Token{pattern.typ, matches[1]})
  46. }
  47. pos = pos + len(matches[0])
  48. break
  49. }
  50. }
  51. }
  52. return
  53. }
  54. // Expand until there are no more expansions to do
  55. func (tokens Tokens) Expand() (result Tokens, err error) {
  56. var updated bool
  57. for i := 0; i < len(tokens); i++ {
  58. var start int
  59. quote := Token{symbolToken, "'"}
  60. if *tokens[i] != quote {
  61. result = append(result, tokens[i])
  62. } else {
  63. updated = true
  64. for start = i + 1; *tokens[start] == quote; start++ {
  65. result = append(result, tokens[start])
  66. }
  67. if tokens[i+1].typ == openToken {
  68. if i, err = tokens.findClose(start + 1); err != nil {
  69. return nil, err
  70. }
  71. } else {
  72. i = start
  73. }
  74. result = append(result, &Token{openToken, "("}, &Token{symbolToken, "quote"})
  75. result = append(result, tokens[start:i+1]...)
  76. result = append(result, &Token{closeToken, ")"})
  77. }
  78. }
  79. if updated {
  80. result, err = result.Expand()
  81. }
  82. return
  83. }
  84. func (tokens Tokens) Parse() (cons Cons, err error) {
  85. var pos int
  86. var current *Cons
  87. for pos < len(tokens) {
  88. if current == nil {
  89. cons = Cons{&Nil, &Nil}
  90. current = &cons
  91. } else {
  92. previous_current := current
  93. current = &Cons{&Nil, &Nil}
  94. previous_current.cdr = &Value{consValue, current}
  95. }
  96. t := tokens[pos]
  97. switch t.typ {
  98. case numberToken:
  99. if i, err := strconv.ParseFloat(t.val, 64); err != nil {
  100. err = fmt.Errorf("Failed to convert number: %v", t.val)
  101. } else {
  102. current.car = &Value{numberValue, i}
  103. pos++
  104. }
  105. case stringToken:
  106. current.car = &Value{stringValue, t.val[1 : len(t.val)-1]}
  107. pos++
  108. case symbolToken:
  109. current.car = &Value{symbolValue, t.val}
  110. pos++
  111. case openToken:
  112. var nested Cons
  113. start := pos + 1
  114. var end int
  115. if end, err = tokens.findClose(start); err != nil {
  116. return
  117. }
  118. if start == end {
  119. current.car = &Nil
  120. } else {
  121. if nested, err = tokens[start:end].Parse(); err != nil {
  122. return
  123. }
  124. current.car = &Value{consValue, &nested}
  125. }
  126. pos = end + 1
  127. case closeToken:
  128. err = fmt.Errorf("List was closed but not opened")
  129. }
  130. }
  131. return
  132. }
  133. func (t Tokens) findClose(start int) (int, error) {
  134. depth := 1
  135. for i := start; i < len(t); i++ {
  136. t := t[i]
  137. switch t.typ {
  138. case openToken:
  139. depth++
  140. case closeToken:
  141. depth--
  142. }
  143. if depth == 0 {
  144. return i, nil
  145. }
  146. }
  147. return 0, fmt.Errorf("List was opened but not closed")
  148. }