Function: hif-tokenize

hif-tokenize is a byte-compiled function defined in hideif.el.gz.

Signature

(hif-tokenize START END)

Documentation

Separate string between START and END into a list of tokens.

Source Code

;; Defined in /usr/src/emacs/lisp/progmodes/hideif.el.gz
(defun hif-tokenize (start end)
  "Separate string between START and END into a list of tokens."
  (let ((token-list nil)
        (white-regexp "[ \t]+")
        token)
    (setq hif-simple-token-only t)
    (with-syntax-table hide-ifdef-syntax-table
      (save-excursion
        (save-restriction
          ;; Narrow down to the focusing region so that the ending white spaces
          ;; of that line will not be treated as a white, as `looking-at' won't
          ;; look outside the restriction; otherwise it will note the last token
          ;; or string as one with an `hif-space' property.
          (setq end (hif-backward-comment start end))
          (narrow-to-region start end)
          (goto-char start)
          (while (progn (forward-comment (point-max)) (< (point) end))
            ;; (message "expr-start = %d" expr-start) (sit-for 1)
            (cond
             ((looking-at "\\\\\n")
              (forward-char 2))

             ((looking-at hif-string-literal-regexp)
              (setq token (substring-no-properties (match-string 1)))
              (goto-char (match-end 0))
              (when (looking-at white-regexp)
                (add-text-properties 0 1 '(hif-space t) token)
                (goto-char (match-end 0)))
              (push token token-list))

             ((looking-at hif-token-regexp)
              (goto-char (match-end 0))
              (setq token (hif-strtok
                           (substring-no-properties (match-string 0))))
              (push token token-list)
              (when (looking-at white-regexp)
                ;; We can't just append a space to the token string, otherwise
                ;; `0xf0 ' ## `01' will become `0xf0 01' instead of the expected
                ;; `0xf001', hence a standalone `hif-space' is placed instead.
                (push 'hif-space token-list)
                (goto-char (match-end 0))))

             ((looking-at "\r") ; Sometimes MS-Windows user will leave CR in
              (forward-char 1)) ;  the source code. Let's not get stuck here.

             (t (error "Bad #if expression: %s" (buffer-string)))))))
      (if (eq 'hif-space (car token-list))
          (setq token-list (cdr token-list))) ;; remove trailing white space
      (nreverse token-list))))