2016-05-17 23:55:38 +03:00
|
|
|
|
;;; char-fold-tests.el --- Tests for char-fold.el -*- lexical-binding: t; -*-
|
2015-10-30 12:18:46 +00:00
|
|
|
|
|
2018-01-01 00:21:42 -08:00
|
|
|
|
;; Copyright (C) 2013-2018 Free Software Foundation, Inc.
|
2015-10-30 12:18:46 +00:00
|
|
|
|
|
|
|
|
|
;; Author: Artur Malabarba <bruce.connor.am@gmail.com>
|
|
|
|
|
|
|
|
|
|
;; This program is free software; you can redistribute it and/or modify
|
|
|
|
|
;; it under the terms of the GNU General Public License as published by
|
|
|
|
|
;; the Free Software Foundation, either version 3 of the License, or
|
|
|
|
|
;; (at your option) any later version.
|
|
|
|
|
|
|
|
|
|
;; This program is distributed in the hope that it will be useful,
|
|
|
|
|
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
;; GNU General Public License for more details.
|
|
|
|
|
|
|
|
|
|
;; You should have received a copy of the GNU General Public License
|
2017-09-13 15:52:52 -07:00
|
|
|
|
;; along with this program. If not, see <https://www.gnu.org/licenses/>.
|
2015-10-30 12:18:46 +00:00
|
|
|
|
|
|
|
|
|
;;; Code:
|
|
|
|
|
|
|
|
|
|
(require 'ert)
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(require 'char-fold)
|
2015-10-30 12:18:46 +00:00
|
|
|
|
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(defun char-fold--random-word (n)
|
2015-10-30 12:18:46 +00:00
|
|
|
|
(mapconcat (lambda (_) (string (+ 9 (random 117))))
|
|
|
|
|
(make-list n nil) ""))
|
|
|
|
|
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(defun char-fold--test-search-with-contents (contents string)
|
2015-10-30 12:18:46 +00:00
|
|
|
|
(with-temp-buffer
|
|
|
|
|
(insert contents)
|
|
|
|
|
(goto-char (point-min))
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(should (search-forward-regexp (char-fold-to-regexp string) nil 'noerror))
|
2015-10-30 12:18:46 +00:00
|
|
|
|
(goto-char (point-min))
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(should (char-fold-search-forward string nil 'noerror))
|
|
|
|
|
(should (char-fold-search-backward string nil 'noerror))))
|
2015-10-30 12:18:46 +00:00
|
|
|
|
|
|
|
|
|
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(ert-deftest char-fold--test-consistency ()
|
2015-11-28 19:56:19 +00:00
|
|
|
|
(dotimes (n 30)
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(let ((w (char-fold--random-word n)))
|
2015-10-30 12:18:46 +00:00
|
|
|
|
;; A folded string should always match the original string.
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(char-fold--test-search-with-contents w w))))
|
2015-10-30 12:18:46 +00:00
|
|
|
|
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(ert-deftest char-fold--test-lax-whitespace ()
|
2015-11-28 19:56:19 +00:00
|
|
|
|
(dotimes (n 40)
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(let ((w1 (char-fold--random-word n))
|
|
|
|
|
(w2 (char-fold--random-word n))
|
2015-10-30 12:18:46 +00:00
|
|
|
|
(search-spaces-regexp "\\s-+"))
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(char-fold--test-search-with-contents
|
2015-10-30 12:18:46 +00:00
|
|
|
|
(concat w1 "\s\n\s\t\f\t\n\r\t" w2)
|
|
|
|
|
(concat w1 " " w2))
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(char-fold--test-search-with-contents
|
2015-10-30 12:18:46 +00:00
|
|
|
|
(concat w1 "\s\n\s\t\f\t\n\r\t" w2)
|
2015-11-28 12:15:17 +00:00
|
|
|
|
(concat w1 (make-string 10 ?\s) w2)))))
|
|
|
|
|
|
Support casing characters which map into multiple code points (bug#24603)
Implement unconditional special casing rules defined in Unicode standard.
Among other things, they deal with cases when a single code point is
replaced by multiple ones because single character does not exist (e.g.
‘fi’ ligature turning into ‘FL’) or is not commonly used (e.g. ß turning
into SS).
* admin/unidata/SpecialCasing.txt: New data file pulled from Unicode
standard distribution.
* admin/unidata/README: Mention SpecialCasing.txt.
* admin/unidata/unidata-get.el (unidata-gen-table-special-casing,
unidata-gen-table-special-casing--do-load): New functions generating
‘special-uppercase’, ‘special-lowercase’ and ‘special-titlecase’
character Unicode properties built from the SpecialCasing.txt Unicode
data file.
* src/casefiddle.c (struct casing_str_buf): New structure for
representing short strings used to handle one-to-many character
mappings.
(case_character_imlp): New function which can handle one-to-many
character mappings.
(case_character, case_single_character): Wrappers for the above
functions. The former may map one character to multiple (or no)
code points while the latter does what the former used to do (i.e.
handles one-to-one mappings only).
(do_casify_natnum, do_casify_unibyte_string,
do_casify_unibyte_region): Use case_single_character.
(do_casify_multibyte_string, do_casify_multibyte_region): Support new
features of case_character.
* (do_casify_region): Updated to reflact do_casify_multibyte_string
changes.
(casify_word): Handle situation when one character-length of a word
can change affecting where end of the word is.
(upcase, capitalize, upcase-initials): Update documentation to mention
limitations when working on characters.
* test/src/casefiddle-tests.el (casefiddle-tests-char-properties):
Add test cases for the newly introduced character properties.
(casefiddle-tests-casing): Update test cases which are now passing.
* test/lisp/char-fold-tests.el (char-fold--ascii-upcase,
char-fold--ascii-downcase): New functions which behave like old ‘upcase’
and ‘downcase’.
(char-fold--test-match-exactly): Use the new functions. This is needed
because otherwise fi and similar characters are turned into their multi-
-character representation.
* doc/lispref/strings.texi: Describe issue with casing characters versus
strings.
* doc/lispref/nonascii.texi: Describe the new character properties.
2016-10-05 00:06:01 +02:00
|
|
|
|
(defun char-fold--ascii-upcase (string)
|
|
|
|
|
"Like `upcase' but acts on ASCII characters only."
|
|
|
|
|
(replace-regexp-in-string "[a-z]+" 'upcase string))
|
|
|
|
|
|
|
|
|
|
(defun char-fold--ascii-downcase (string)
|
|
|
|
|
"Like `downcase' but acts on ASCII characters only."
|
|
|
|
|
(replace-regexp-in-string "[a-z]+" 'downcase string))
|
|
|
|
|
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(defun char-fold--test-match-exactly (string &rest strings-to-match)
|
|
|
|
|
(let ((re (concat "\\`" (char-fold-to-regexp string) "\\'")))
|
2015-11-28 12:15:17 +00:00
|
|
|
|
(dolist (it strings-to-match)
|
2015-11-28 15:31:43 +00:00
|
|
|
|
(should (string-match re it)))
|
|
|
|
|
;; Case folding
|
|
|
|
|
(let ((case-fold-search t))
|
|
|
|
|
(dolist (it strings-to-match)
|
Support casing characters which map into multiple code points (bug#24603)
Implement unconditional special casing rules defined in Unicode standard.
Among other things, they deal with cases when a single code point is
replaced by multiple ones because single character does not exist (e.g.
‘fi’ ligature turning into ‘FL’) or is not commonly used (e.g. ß turning
into SS).
* admin/unidata/SpecialCasing.txt: New data file pulled from Unicode
standard distribution.
* admin/unidata/README: Mention SpecialCasing.txt.
* admin/unidata/unidata-get.el (unidata-gen-table-special-casing,
unidata-gen-table-special-casing--do-load): New functions generating
‘special-uppercase’, ‘special-lowercase’ and ‘special-titlecase’
character Unicode properties built from the SpecialCasing.txt Unicode
data file.
* src/casefiddle.c (struct casing_str_buf): New structure for
representing short strings used to handle one-to-many character
mappings.
(case_character_imlp): New function which can handle one-to-many
character mappings.
(case_character, case_single_character): Wrappers for the above
functions. The former may map one character to multiple (or no)
code points while the latter does what the former used to do (i.e.
handles one-to-one mappings only).
(do_casify_natnum, do_casify_unibyte_string,
do_casify_unibyte_region): Use case_single_character.
(do_casify_multibyte_string, do_casify_multibyte_region): Support new
features of case_character.
* (do_casify_region): Updated to reflact do_casify_multibyte_string
changes.
(casify_word): Handle situation when one character-length of a word
can change affecting where end of the word is.
(upcase, capitalize, upcase-initials): Update documentation to mention
limitations when working on characters.
* test/src/casefiddle-tests.el (casefiddle-tests-char-properties):
Add test cases for the newly introduced character properties.
(casefiddle-tests-casing): Update test cases which are now passing.
* test/lisp/char-fold-tests.el (char-fold--ascii-upcase,
char-fold--ascii-downcase): New functions which behave like old ‘upcase’
and ‘downcase’.
(char-fold--test-match-exactly): Use the new functions. This is needed
because otherwise fi and similar characters are turned into their multi-
-character representation.
* doc/lispref/strings.texi: Describe issue with casing characters versus
strings.
* doc/lispref/nonascii.texi: Describe the new character properties.
2016-10-05 00:06:01 +02:00
|
|
|
|
(should (string-match (char-fold--ascii-upcase re) (downcase it)))
|
|
|
|
|
(should (string-match (char-fold--ascii-downcase re) (upcase it)))))))
|
2015-11-28 12:15:17 +00:00
|
|
|
|
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(ert-deftest char-fold--test-some-defaults ()
|
2015-11-28 12:15:17 +00:00
|
|
|
|
(dolist (it '(("ffl" . "ffl") ("ffi" . "ffi")
|
|
|
|
|
("fi" . "fi") ("ff" . "ff")
|
|
|
|
|
("ä" . "ä")))
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(char-fold--test-search-with-contents (cdr it) (car it))
|
|
|
|
|
(let ((multi (char-table-extra-slot char-fold-table 0))
|
|
|
|
|
(char-fold-table (make-char-table 'char-fold-table)))
|
|
|
|
|
(set-char-table-extra-slot char-fold-table 0 multi)
|
|
|
|
|
(char-fold--test-match-exactly (car it) (cdr it)))))
|
|
|
|
|
|
|
|
|
|
(ert-deftest char-fold--test-fold-to-regexp ()
|
|
|
|
|
(let ((char-fold-table (make-char-table 'char-fold-table))
|
|
|
|
|
(multi (make-char-table 'char-fold-table)))
|
|
|
|
|
(set-char-table-extra-slot char-fold-table 0 multi)
|
|
|
|
|
(aset char-fold-table ?a "xx")
|
|
|
|
|
(aset char-fold-table ?1 "44")
|
|
|
|
|
(aset char-fold-table ?\s "-!-")
|
|
|
|
|
(char-fold--test-match-exactly "a1a1" "xx44xx44")
|
|
|
|
|
(char-fold--test-match-exactly "a1 a 1" "xx44-!--!-xx-!-44")
|
2015-11-28 12:15:17 +00:00
|
|
|
|
(aset multi ?a '(("1" . "99")
|
|
|
|
|
("2" . "88")
|
|
|
|
|
("12" . "77")))
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(char-fold--test-match-exactly "a" "xx")
|
|
|
|
|
(char-fold--test-match-exactly "a1" "xx44" "99")
|
|
|
|
|
(char-fold--test-match-exactly "a12" "77" "xx442" "992")
|
|
|
|
|
(char-fold--test-match-exactly "a2" "88")
|
2015-11-28 12:15:17 +00:00
|
|
|
|
(aset multi ?1 '(("2" . "yy")))
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(char-fold--test-match-exactly "a1" "xx44" "99")
|
|
|
|
|
(char-fold--test-match-exactly "a12" "77" "xx442" "992")
|
2015-12-01 13:52:50 +00:00
|
|
|
|
;; Support for this case is disabled. See function definition or:
|
2017-11-25 22:45:41 -08:00
|
|
|
|
;; https://lists.gnu.org/r/emacs-devel/2015-11/msg02562.html
|
2016-05-17 23:55:38 +03:00
|
|
|
|
;; (char-fold--test-match-exactly "a12" "xxyy")
|
2015-12-01 13:52:50 +00:00
|
|
|
|
))
|
2015-11-28 12:15:17 +00:00
|
|
|
|
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(ert-deftest char-fold--speed-test ()
|
2015-12-04 15:12:10 +00:00
|
|
|
|
(dolist (string (append '("tty-set-up-initial-frame-face"
|
|
|
|
|
"tty-set-up-initial-frame-face-frame-faceframe-faceframe-faceframe-face")
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(mapcar #'char-fold--random-word '(10 50 100
|
2015-12-04 15:12:10 +00:00
|
|
|
|
50 100))))
|
|
|
|
|
(message "Testing %s" string)
|
|
|
|
|
;; Make sure we didn't just fallback on the trivial search.
|
|
|
|
|
(should-not (string= (regexp-quote string)
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(char-fold-to-regexp string)))
|
2015-12-04 15:12:10 +00:00
|
|
|
|
(with-temp-buffer
|
|
|
|
|
(save-excursion (insert string))
|
Prefer nil to (current-time) when either works
* doc/misc/gnus.texi (Category Syntax):
* lisp/allout-widgets.el (allout-widgets-post-command-business):
* lisp/cedet/ede/detect.el (ede-detect-qtest):
* lisp/cedet/pulse.el (pulse-momentary-highlight-overlay)
(pulse-tick):
* lisp/cedet/semantic.el (bovinate):
* lisp/cedet/semantic/analyze.el:
(semantic-analyze-current-symbol-default, semantic-adebug-analyze):
* lisp/cedet/semantic/analyze/refs.el (semantic-analyze-current-tag):
* lisp/cedet/semantic/lex.el (semantic-lex-test):
* lisp/cedet/semantic/symref/filter.el:
(semantic-symref-test-count-hits-in-tag):
* lisp/cedet/srecode/dictionary.el (srecode-adebug-dictionary):
* lisp/cedet/srecode/map.el (srecode-adebug-maps):
* lisp/desktop.el (desktop-create-buffer):
* lisp/emacs-lisp/benchmark.el (benchmark-elapse):
* lisp/emacs-lisp/elp.el (elp--make-wrapper):
* lisp/epa.el (epa--show-key):
* lisp/erc/erc.el (erc-lurker-cleanup, erc-lurker-p):
* lisp/gnus/gnus-agent.el (gnus-agent-fetch-articles)
(gnus-agent-expire-group-1, gnus-agent-store-article):
* lisp/gnus/gnus-art.el (article-lapsed-string):
* lisp/gnus/gnus-cloud.el (gnus-cloud-update-newsrc-data)
(gnus-cloud-collect-full-newsrc):
* lisp/gnus/gnus-group.el (gnus-group-timestamp-delta):
* lisp/gnus/gnus-html.el (gnus-html-cache-expired):
* lisp/gnus/gnus-score.el (gnus-score-load-file)
(gnus-decay-scores):
* lisp/gnus/nndiary.el (nndiary-expired-article-p):
* lisp/gnus/nnmail.el (nnmail-expired-article-p):
* lisp/gnus/nnmaildir.el (nnmaildir--scan):
* lisp/gnus/score-mode.el (gnus-score-edit-insert-date):
* lisp/image/gravatar.el (gravatar-cache-expired):
* lisp/net/newst-backend.el (newsticker--image-get)
(newsticker--cache-mark-expired):
* lisp/nxml/rng-maint.el (rng-time-function):
* lisp/org/org-agenda.el (org-agenda-to-appt):
* lisp/org/org-clock.el (org-clock-resolve-clock)
(org-clock-resolve, org-resolve-clocks-if-idle):
* lisp/org/org-colview.el (org-columns-edit-value, org-columns)
(org-columns-compute-all, org-agenda-columns):
* lisp/org/org-element.el (org-element--cache-interrupt-p)
(org-element--cache-sync):
* lisp/org/org-habit.el (org-habit-get-faces)
(org-habit-insert-consistency-graphs):
* lisp/org/org-indent.el (org-indent-add-properties):
* lisp/org/org-timer.el (org-timer-start)
(org-timer-pause-or-continue, org-timer-seconds)
(org-timer-show-remaining-time, org-timer-set-timer):
* lisp/org/org.el (org-babel-load-file, org-current-time)
(org-today, org-auto-repeat-maybe, org-read-date-analyze)
(org-small-year-to-year, org-goto-calendar):
* lisp/org/ox.el (org-export-insert-default-template):
* lisp/time.el (emacs-uptime):
* lisp/type-break.el (type-break-mode, type-break)
(type-break-time-warning-schedule, type-break-check):
* lisp/url/url-cache.el (url-cache-expired):
* lisp/url/url.el (url-retrieve-synchronously):
* test/lisp/char-fold-tests.el (char-fold--speed-test):
* test/manual/cedet/semantic-ia-utest.el:
(semantic-symref-test-count-hits-in-tag):
* test/manual/cedet/semantic-tests.el (semantic-idle-pnf-test)
(semantic-lex-test-full-depth):
Use nil instead of (current-time) where either will do, as nil is
a bit more efficient and should have less timing error.
2017-10-20 19:40:09 -07:00
|
|
|
|
(let ((time (time-to-seconds)))
|
2015-12-04 15:12:10 +00:00
|
|
|
|
;; Our initial implementation of case-folding in char-folding
|
|
|
|
|
;; created a lot of redundant paths in the regexp. Because of
|
|
|
|
|
;; that, if a really long string "almost" matches, the regexp
|
2015-12-07 08:35:53 -08:00
|
|
|
|
;; engine took a long time to realize that it doesn't match.
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(should-not (char-fold-search-forward (concat string "c") nil 'noerror))
|
2015-12-04 15:12:10 +00:00
|
|
|
|
;; Ensure it took less than a second.
|
Prefer nil to (current-time) when either works
* doc/misc/gnus.texi (Category Syntax):
* lisp/allout-widgets.el (allout-widgets-post-command-business):
* lisp/cedet/ede/detect.el (ede-detect-qtest):
* lisp/cedet/pulse.el (pulse-momentary-highlight-overlay)
(pulse-tick):
* lisp/cedet/semantic.el (bovinate):
* lisp/cedet/semantic/analyze.el:
(semantic-analyze-current-symbol-default, semantic-adebug-analyze):
* lisp/cedet/semantic/analyze/refs.el (semantic-analyze-current-tag):
* lisp/cedet/semantic/lex.el (semantic-lex-test):
* lisp/cedet/semantic/symref/filter.el:
(semantic-symref-test-count-hits-in-tag):
* lisp/cedet/srecode/dictionary.el (srecode-adebug-dictionary):
* lisp/cedet/srecode/map.el (srecode-adebug-maps):
* lisp/desktop.el (desktop-create-buffer):
* lisp/emacs-lisp/benchmark.el (benchmark-elapse):
* lisp/emacs-lisp/elp.el (elp--make-wrapper):
* lisp/epa.el (epa--show-key):
* lisp/erc/erc.el (erc-lurker-cleanup, erc-lurker-p):
* lisp/gnus/gnus-agent.el (gnus-agent-fetch-articles)
(gnus-agent-expire-group-1, gnus-agent-store-article):
* lisp/gnus/gnus-art.el (article-lapsed-string):
* lisp/gnus/gnus-cloud.el (gnus-cloud-update-newsrc-data)
(gnus-cloud-collect-full-newsrc):
* lisp/gnus/gnus-group.el (gnus-group-timestamp-delta):
* lisp/gnus/gnus-html.el (gnus-html-cache-expired):
* lisp/gnus/gnus-score.el (gnus-score-load-file)
(gnus-decay-scores):
* lisp/gnus/nndiary.el (nndiary-expired-article-p):
* lisp/gnus/nnmail.el (nnmail-expired-article-p):
* lisp/gnus/nnmaildir.el (nnmaildir--scan):
* lisp/gnus/score-mode.el (gnus-score-edit-insert-date):
* lisp/image/gravatar.el (gravatar-cache-expired):
* lisp/net/newst-backend.el (newsticker--image-get)
(newsticker--cache-mark-expired):
* lisp/nxml/rng-maint.el (rng-time-function):
* lisp/org/org-agenda.el (org-agenda-to-appt):
* lisp/org/org-clock.el (org-clock-resolve-clock)
(org-clock-resolve, org-resolve-clocks-if-idle):
* lisp/org/org-colview.el (org-columns-edit-value, org-columns)
(org-columns-compute-all, org-agenda-columns):
* lisp/org/org-element.el (org-element--cache-interrupt-p)
(org-element--cache-sync):
* lisp/org/org-habit.el (org-habit-get-faces)
(org-habit-insert-consistency-graphs):
* lisp/org/org-indent.el (org-indent-add-properties):
* lisp/org/org-timer.el (org-timer-start)
(org-timer-pause-or-continue, org-timer-seconds)
(org-timer-show-remaining-time, org-timer-set-timer):
* lisp/org/org.el (org-babel-load-file, org-current-time)
(org-today, org-auto-repeat-maybe, org-read-date-analyze)
(org-small-year-to-year, org-goto-calendar):
* lisp/org/ox.el (org-export-insert-default-template):
* lisp/time.el (emacs-uptime):
* lisp/type-break.el (type-break-mode, type-break)
(type-break-time-warning-schedule, type-break-check):
* lisp/url/url-cache.el (url-cache-expired):
* lisp/url/url.el (url-retrieve-synchronously):
* test/lisp/char-fold-tests.el (char-fold--speed-test):
* test/manual/cedet/semantic-ia-utest.el:
(semantic-symref-test-count-hits-in-tag):
* test/manual/cedet/semantic-tests.el (semantic-idle-pnf-test)
(semantic-lex-test-full-depth):
Use nil instead of (current-time) where either will do, as nil is
a bit more efficient and should have less timing error.
2017-10-20 19:40:09 -07:00
|
|
|
|
(should (< (- (time-to-seconds) time) 1))))))
|
2015-11-28 10:32:46 +00:00
|
|
|
|
|
2016-05-17 23:55:38 +03:00
|
|
|
|
(provide 'char-fold-tests)
|
|
|
|
|
;;; char-fold-tests.el ends here
|