Function: hif-tokenize

hif-tokenize is a byte-compiled function defined in hideif.el.gz.

Signature

(hif-tokenize START END)

Documentation

Separate string between START and END into a list of tokens.

Source Code

;; Defined in /usr/src/emacs/lisp/progmodes/hideif.el.gz
(defun hif-tokenize (start end)
  "Separate string between START and END into a list of tokens."
  (let ((token-list nil)
        ;; Similar to `hif-white-regexp' but keep the spaces if there are
        (white-regexp (concat "\\(?:"
                              "\\([ \t]+\\)\\|/\\*.*?\\*/"
                              "\\|\\(?:" hif-line-concat "\\)"
                              "\\)*"))
        token)
    (setq hif-simple-token-only t)
    (with-syntax-table hide-ifdef-syntax-table
      (save-excursion
        (save-restriction
          ;; Narrow down to the focusing region so that the ending white spaces
          ;; of that line will not be treated as a white, as `looking-at' won't
          ;; look outside the restriction; otherwise it will note the last token
          ;; or string as one with an `hif-space' property.
          (setq end (hif-backward-comment start end))
          (narrow-to-region start end)
          (goto-char start)
          (while (progn (forward-comment (point-max)) (< (point) end))
            ;; (message "expr-start = %d" expr-start) (sit-for 1)
            (cond
             ((looking-at "\\\\\n")
              (forward-char 2))

             ((looking-at hif-string-literal-regexp)
              (setq token (match-string-no-properties 1))
              (goto-char (match-end 0))
              (when (looking-at white-regexp)
                (if (not (zerop (length (match-string-no-properties 1))))
                    (add-text-properties 0 1 '(hif-space t) token))
                (goto-char (match-end 0)))
              (push token token-list))

             ((looking-at hif-token-regexp)
              (goto-char (match-end 0))
              (setq token (hif-strtok (match-string-no-properties 0)))
              (push token token-list)
              (when (looking-at white-regexp)
                (if (not (zerop (length (match-string-no-properties 1))))
                    ;; We can't just append a space to the token string,
                    ;; otherwise `0xf0 ' ## `01' will become `0xf0 01' instead
                    ;; of the expected `0xf001', hence a standalone `hif-space'
                    ;; is placed instead.
                    (push 'hif-space token-list))
                (goto-char (match-end 0))))

             ((looking-at "\r") ; Sometimes MS-Windows user will leave CR in
              (forward-char 1)) ;  the source code. Let's not get stuck here.

             (t (error "Bad preprocessor expression: %s" (buffer-string)))))))
      (if (eq 'hif-space (car token-list))
          (setq token-list (cdr token-list))) ;; remove trailing white space
      (nreverse token-list))))