shlex.go 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458
  1. /*
  2. Copyright 2012 Google Inc. All Rights Reserved.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package shlex
  14. /*
  15. Package shlex implements a simple lexer which splits input in to tokens using
  16. shell-style rules for quoting and commenting.
  17. */
  18. import (
  19. "bufio"
  20. "errors"
  21. "fmt"
  22. "io"
  23. "strings"
  24. )
  25. /*
  26. A TokenType is a top-level token; a word, space, comment, unknown.
  27. */
  28. type TokenType int
  29. /*
  30. A RuneTokenType is the type of a UTF-8 character; a character, quote, space, escape.
  31. */
  32. type RuneTokenType int
  33. type lexerState int
  34. type Token struct {
  35. tokenType TokenType
  36. value string
  37. }
  38. /*
  39. Two tokens are equal if both their types and values are equal. A nil token can
  40. never equal another token.
  41. */
  42. func (a *Token) Equal(b *Token) bool {
  43. if a == nil || b == nil {
  44. return false
  45. }
  46. if a.tokenType != b.tokenType {
  47. return false
  48. }
  49. return a.value == b.value
  50. }
  51. const (
  52. RUNE_CHAR string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-,/@$*()+=><:;&^%~|!?[]{}"
  53. RUNE_SPACE string = " \t\r\n"
  54. RUNE_ESCAPING_QUOTE string = "\""
  55. RUNE_NONESCAPING_QUOTE string = "'"
  56. RUNE_ESCAPE = "\\"
  57. RUNE_COMMENT = "#"
  58. RUNETOKEN_UNKNOWN RuneTokenType = 0
  59. RUNETOKEN_CHAR RuneTokenType = 1
  60. RUNETOKEN_SPACE RuneTokenType = 2
  61. RUNETOKEN_ESCAPING_QUOTE RuneTokenType = 3
  62. RUNETOKEN_NONESCAPING_QUOTE RuneTokenType = 4
  63. RUNETOKEN_ESCAPE RuneTokenType = 5
  64. RUNETOKEN_COMMENT RuneTokenType = 6
  65. RUNETOKEN_EOF RuneTokenType = 7
  66. TOKEN_UNKNOWN TokenType = 0
  67. TOKEN_WORD TokenType = 1
  68. TOKEN_SPACE TokenType = 2
  69. TOKEN_COMMENT TokenType = 3
  70. STATE_START lexerState = 0
  71. STATE_INWORD lexerState = 1
  72. STATE_ESCAPING lexerState = 2
  73. STATE_ESCAPING_QUOTED lexerState = 3
  74. STATE_QUOTED_ESCAPING lexerState = 4
  75. STATE_QUOTED lexerState = 5
  76. STATE_COMMENT lexerState = 6
  77. INITIAL_TOKEN_CAPACITY int = 100
  78. )
  79. /*
  80. A type for classifying characters. This allows for different sorts of
  81. classifiers - those accepting extended non-ascii chars, or strict posix
  82. compatibility, for example.
  83. */
  84. type TokenClassifier struct {
  85. typeMap map[int32]RuneTokenType
  86. }
  87. func addRuneClass(typeMap *map[int32]RuneTokenType, runes string, tokenType RuneTokenType) {
  88. for _, rune := range runes {
  89. (*typeMap)[int32(rune)] = tokenType
  90. }
  91. }
  92. /*
  93. Create a new classifier for basic ASCII characters.
  94. */
  95. func NewDefaultClassifier() *TokenClassifier {
  96. typeMap := map[int32]RuneTokenType{}
  97. addRuneClass(&typeMap, RUNE_CHAR, RUNETOKEN_CHAR)
  98. addRuneClass(&typeMap, RUNE_SPACE, RUNETOKEN_SPACE)
  99. addRuneClass(&typeMap, RUNE_ESCAPING_QUOTE, RUNETOKEN_ESCAPING_QUOTE)
  100. addRuneClass(&typeMap, RUNE_NONESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE)
  101. addRuneClass(&typeMap, RUNE_ESCAPE, RUNETOKEN_ESCAPE)
  102. addRuneClass(&typeMap, RUNE_COMMENT, RUNETOKEN_COMMENT)
  103. return &TokenClassifier{
  104. typeMap: typeMap}
  105. }
  106. func (classifier *TokenClassifier) ClassifyRune(rune int32) RuneTokenType {
  107. return classifier.typeMap[rune]
  108. }
  109. /*
  110. A type for turning an input stream in to a sequence of strings. Whitespace and
  111. comments are skipped.
  112. */
  113. type Lexer struct {
  114. tokenizer *Tokenizer
  115. }
  116. /*
  117. Create a new lexer.
  118. */
  119. func NewLexer(r io.Reader) (*Lexer, error) {
  120. tokenizer, err := NewTokenizer(r)
  121. if err != nil {
  122. return nil, err
  123. }
  124. lexer := &Lexer{tokenizer: tokenizer}
  125. return lexer, nil
  126. }
  127. /*
  128. Return the next word, and an error value. If there are no more words, the error
  129. will be io.EOF.
  130. */
  131. func (l *Lexer) NextWord() (string, error) {
  132. var token *Token
  133. var err error
  134. for {
  135. token, err = l.tokenizer.NextToken()
  136. if err != nil {
  137. return "", err
  138. }
  139. switch token.tokenType {
  140. case TOKEN_WORD:
  141. {
  142. return token.value, nil
  143. }
  144. case TOKEN_COMMENT:
  145. {
  146. // skip comments
  147. }
  148. default:
  149. {
  150. panic(fmt.Sprintf("Unknown token type: %v", token.tokenType))
  151. }
  152. }
  153. }
  154. return "", io.EOF
  155. }
  156. /*
  157. A type for turning an input stream in to a sequence of typed tokens.
  158. */
  159. type Tokenizer struct {
  160. input *bufio.Reader
  161. classifier *TokenClassifier
  162. }
  163. /*
  164. Create a new tokenizer.
  165. */
  166. func NewTokenizer(r io.Reader) (*Tokenizer, error) {
  167. input := bufio.NewReader(r)
  168. classifier := NewDefaultClassifier()
  169. tokenizer := &Tokenizer{
  170. input: input,
  171. classifier: classifier}
  172. return tokenizer, nil
  173. }
  174. /*
  175. Scan the stream for the next token.
  176. This uses an internal state machine. It will panic if it encounters a character
  177. which it does not know how to handle.
  178. */
  179. func (t *Tokenizer) scanStream() (*Token, error) {
  180. state := STATE_START
  181. var tokenType TokenType
  182. value := make([]int32, 0, INITIAL_TOKEN_CAPACITY)
  183. var (
  184. nextRune int32
  185. nextRuneType RuneTokenType
  186. err error
  187. )
  188. SCAN:
  189. for {
  190. nextRune, _, err = t.input.ReadRune()
  191. nextRuneType = t.classifier.ClassifyRune(nextRune)
  192. if err != nil {
  193. if err == io.EOF {
  194. nextRuneType = RUNETOKEN_EOF
  195. err = nil
  196. } else {
  197. return nil, err
  198. }
  199. }
  200. switch state {
  201. case STATE_START: // no runes read yet
  202. {
  203. switch nextRuneType {
  204. case RUNETOKEN_EOF:
  205. {
  206. return nil, io.EOF
  207. }
  208. case RUNETOKEN_CHAR:
  209. {
  210. tokenType = TOKEN_WORD
  211. value = append(value, nextRune)
  212. state = STATE_INWORD
  213. }
  214. case RUNETOKEN_SPACE:
  215. {
  216. }
  217. case RUNETOKEN_ESCAPING_QUOTE:
  218. {
  219. tokenType = TOKEN_WORD
  220. state = STATE_QUOTED_ESCAPING
  221. }
  222. case RUNETOKEN_NONESCAPING_QUOTE:
  223. {
  224. tokenType = TOKEN_WORD
  225. state = STATE_QUOTED
  226. }
  227. case RUNETOKEN_ESCAPE:
  228. {
  229. tokenType = TOKEN_WORD
  230. state = STATE_ESCAPING
  231. }
  232. case RUNETOKEN_COMMENT:
  233. {
  234. tokenType = TOKEN_COMMENT
  235. state = STATE_COMMENT
  236. }
  237. default:
  238. {
  239. return nil, errors.New(fmt.Sprintf("Unknown rune: %v", nextRune))
  240. }
  241. }
  242. }
  243. case STATE_INWORD: // in a regular word
  244. {
  245. switch nextRuneType {
  246. case RUNETOKEN_EOF:
  247. {
  248. break SCAN
  249. }
  250. case RUNETOKEN_CHAR, RUNETOKEN_COMMENT:
  251. {
  252. value = append(value, nextRune)
  253. }
  254. case RUNETOKEN_SPACE:
  255. {
  256. t.input.UnreadRune()
  257. break SCAN
  258. }
  259. case RUNETOKEN_ESCAPING_QUOTE:
  260. {
  261. state = STATE_QUOTED_ESCAPING
  262. }
  263. case RUNETOKEN_NONESCAPING_QUOTE:
  264. {
  265. state = STATE_QUOTED
  266. }
  267. case RUNETOKEN_ESCAPE:
  268. {
  269. state = STATE_ESCAPING
  270. }
  271. default:
  272. {
  273. return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
  274. }
  275. }
  276. }
  277. case STATE_ESCAPING: // the next rune after an escape character
  278. {
  279. switch nextRuneType {
  280. case RUNETOKEN_EOF:
  281. {
  282. err = errors.New("EOF found after escape character")
  283. break SCAN
  284. }
  285. case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
  286. {
  287. state = STATE_INWORD
  288. value = append(value, nextRune)
  289. }
  290. default:
  291. {
  292. return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
  293. }
  294. }
  295. }
  296. case STATE_ESCAPING_QUOTED: // the next rune after an escape character, in double quotes
  297. {
  298. switch nextRuneType {
  299. case RUNETOKEN_EOF:
  300. {
  301. err = errors.New("EOF found after escape character")
  302. break SCAN
  303. }
  304. case RUNETOKEN_CHAR, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
  305. {
  306. state = STATE_QUOTED_ESCAPING
  307. value = append(value, nextRune)
  308. }
  309. default:
  310. {
  311. return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
  312. }
  313. }
  314. }
  315. case STATE_QUOTED_ESCAPING: // in escaping double quotes
  316. {
  317. switch nextRuneType {
  318. case RUNETOKEN_EOF:
  319. {
  320. err = errors.New("EOF found when expecting closing quote.")
  321. break SCAN
  322. }
  323. case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_NONESCAPING_QUOTE, RUNETOKEN_COMMENT:
  324. {
  325. value = append(value, nextRune)
  326. }
  327. case RUNETOKEN_ESCAPING_QUOTE:
  328. {
  329. state = STATE_INWORD
  330. }
  331. case RUNETOKEN_ESCAPE:
  332. {
  333. state = STATE_ESCAPING_QUOTED
  334. }
  335. default:
  336. {
  337. return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
  338. }
  339. }
  340. }
  341. case STATE_QUOTED: // in non-escaping single quotes
  342. {
  343. switch nextRuneType {
  344. case RUNETOKEN_EOF:
  345. {
  346. err = errors.New("EOF found when expecting closing quote.")
  347. break SCAN
  348. }
  349. case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_SPACE, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT:
  350. {
  351. value = append(value, nextRune)
  352. }
  353. case RUNETOKEN_NONESCAPING_QUOTE:
  354. {
  355. state = STATE_INWORD
  356. }
  357. default:
  358. {
  359. return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
  360. }
  361. }
  362. }
  363. case STATE_COMMENT:
  364. {
  365. switch nextRuneType {
  366. case RUNETOKEN_EOF:
  367. {
  368. break SCAN
  369. }
  370. case RUNETOKEN_CHAR, RUNETOKEN_UNKNOWN, RUNETOKEN_ESCAPING_QUOTE, RUNETOKEN_ESCAPE, RUNETOKEN_COMMENT, RUNETOKEN_NONESCAPING_QUOTE:
  371. {
  372. value = append(value, nextRune)
  373. }
  374. case RUNETOKEN_SPACE:
  375. {
  376. if nextRune == '\n' {
  377. state = STATE_START
  378. break SCAN
  379. } else {
  380. value = append(value, nextRune)
  381. }
  382. }
  383. default:
  384. {
  385. return nil, errors.New(fmt.Sprintf("Uknown rune: %v", nextRune))
  386. }
  387. }
  388. }
  389. default:
  390. {
  391. panic(fmt.Sprintf("Unexpected state: %v", state))
  392. }
  393. }
  394. }
  395. token := &Token{
  396. tokenType: tokenType,
  397. value: string(value)}
  398. return token, err
  399. }
  400. /*
  401. Return the next token in the stream, and an error value. If there are no more
  402. tokens available, the error value will be io.EOF.
  403. */
  404. func (t *Tokenizer) NextToken() (*Token, error) {
  405. return t.scanStream()
  406. }
  407. /*
  408. Split a string in to a slice of strings, based upon shell-style rules for
  409. quoting, escaping, and spaces.
  410. */
  411. func Split(s string) ([]string, error) {
  412. l, err := NewLexer(strings.NewReader(s))
  413. if err != nil {
  414. return nil, err
  415. }
  416. subStrings := []string{}
  417. for {
  418. word, err := l.NextWord()
  419. if err != nil {
  420. if err == io.EOF {
  421. return subStrings, nil
  422. }
  423. return subStrings, err
  424. }
  425. subStrings = append(subStrings, word)
  426. }
  427. return subStrings, nil
  428. }