(define assignment-tok (string->symbol "="))
(define plus-tok (string->symbol "+"))
(define minus-tok (string->symbol "-"))
(define times-tok (string->symbol "*"))
(define div-tok (string->symbol "/"))
(define mod-tok (string->symbol "%"))
(define bit-and-tok (string->symbol "&"))
(define bit-or-tok (string->symbol "|"))
(define bit-xor-tok (string->symbol "^"))
(define shr-tok (string->symbol ">>"))
(define shl-tok (string->symbol "<<"))
(define and-tok (string->symbol "&&"))
(define or-tok (string->symbol "||"))
(define equal-tok (string->symbol "=="))
(define not-equals-tok (string->symbol "!="))
(define less-than-tok (string->symbol "<"))
(define greater-than-tok (string->symbol ">"))
(define less-or-equal-tok (string->symbol "<="))
(define greater-or-equal-tok (string->symbol ">="))
(define unary-minus-tok (string->symbol "-"))
(define bit-not-tok (string->symbol "~"))
(define not-tok (string->symbol "!"))
(define semi-colon-tok (string->symbol ";"))
(define colon-tok (string->symbol ":"))
(define open-paren-tok (string->symbol "("))
(define close-paren-tok (string->symbol ")"))
(define open-bracket-tok (string->symbol "["))
(define close-bracket-tok (string->symbol "]"))
(define period-tok (string->symbol "."))
(define comma-tok (string->symbol ","))
(define include-tok (string->symbol "include"))
(define define-tok (string->symbol "define"))
(define pound-tok (string->symbol "#"))
(define dot-tok (string->symbol ".")) ; only used in include file name
(define integer-tok (string->symbol "int"))
(define void-tok (string->symbol "void"))
(define begin-tok (string->symbol "{"))
(define end-tok (string->symbol "}"))
(define if-tok (string->symbol "if"))
(define else-tok (string->symbol "else"))
(define switch-tok (string->symbol "switch"))
(define while-tok (string->symbol "while"))
(define do-tok (string->symbol "do"))
(define for-tok (string->symbol "for"))
(define case-tok (string->symbol "case"))
(define break-tok (string->symbol "break"))
(define return-tok (string->symbol "return"))
(define printf-tok (string->symbol "printf"))
(define succ-tok (string->symbol "++"))
(define pred-tok (string->symbol "--"))

(define (initialize-lex)
  ; prime the pump
  (get-token)
  (get-token)
  (get-token))

(define (lookahead-var)
  (and (eq? (lookahead) integer-tok)
       (or (eq? (triple-lookahead) semi-colon-tok)
	   (eq? (triple-lookahead) open-bracket-tok))))

(define (lookahead-procedure)
  (and (eq? (lookahead) void-tok)
       (eq? (triple-lookahead) open-paren-tok)))

(define (lookahead-function)
  (and (eq? (lookahead) integer-tok)
       (eq? (triple-lookahead) open-paren-tok)))

; TODO unify these with ones in shunt
; TODO most of the seperator-...-tok can use the literal symbol
(define (binary-operator? token)
  (or (eq? token bit-or-tok)
      (eq? token or-tok)
      (memq token '(
        = + - * / % && << >> & ^ == != < <= >= >
        ))))

(define (operator? token)
  (or
    (binary-operator? token)
    (eq? token unary-minus-tok)
    (eq? token bit-not-tok)
    (eq? token not-tok)))

(define (identifier? token)
  (not (or
    (operator? token)
    (eq? token semi-colon-tok) (eq? token colon-tok)
    (eq? token open-paren-tok) (eq? token close-paren-tok)
    (eq? token open-bracket-tok) (eq? token close-bracket-tok)
    (eq? token period-tok) (eq? token comma-tok))))

; hash table operations

(define HASH-SIZE 509) ; some prime number

(define (make-hash)
  (make-vector HASH-SIZE '()))

(define (hash-set! vec key value)
  (let
    ((hash-idx (remainder (symbol-hash key) HASH-SIZE)))
    (vector-set!
      vec
      hash-idx
      (cons (list key value) (vector-ref vec hash-idx)))))
 
(define (hash-ref vec key)
  (let
    ((hash-idx (remainder (symbol-hash key) HASH-SIZE)))
    (assq key (vector-ref vec hash-idx))))

(define (empty-hash-table? vec n)
  (if (= n HASH-SIZE)
    #t
    (let
      ((element (vector-ref vec n)))
      (if (not (null? element))
        #f
        (empty-hash-table? vec (+ n 1))))))
