/usr/lib/X11/xedit/lisp/syntax.lsp is in x11-apps 7.6+5ubuntu1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 | ;;
;; Copyright (c) 2002 by The XFree86 Project, Inc.
;;
;; Permission is hereby granted, free of charge, to any person obtaining a
;; copy of this software and associated documentation files (the "Software"),
;; to deal in the Software without restriction, including without limitation
;; the rights to use, copy, modify, merge, publish, distribute, sublicense,
;; and/or sell copies of the Software, and to permit persons to whom the
;; Software is furnished to do so, subject to the following conditions:
;;
;; The above copyright notice and this permission notice shall be included in
;; all copies or substantial portions of the Software.
;;
;; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
;; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
;; FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
;; THE XFREE86 PROJECT BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
;; WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
;; OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
;; SOFTWARE.
;;
;; Except as contained in this notice, the name of the XFree86 Project shall
;; not be used in advertising or otherwise to promote the sale, use or other
;; dealings in this Software without prior written authorization from the
;; XFree86 Project.
;;
;; Author: Paulo César Pereira de Andrade
;;
;;
;; $XFree86: xc/programs/xedit/lisp/modules/syntax.lsp,v 1.11 2003/01/16 03:50:46 paulo Exp $
;;
(provide "syntax")
(require "xedit")
(in-package "XEDIT")
(defvar *syntax-symbols* '(
syntax-highlight defsyntax defsynprop synprop-p syntax-p
syntable syntoken synaugment
*prop-default* *prop-keyword* *prop-number* *prop-string*
*prop-constant* *prop-comment* *prop-preprocessor*
*prop-punctuation* *prop-error* *prop-annotation*
))
(export *syntax-symbols*)
(in-package "USER")
(dolist (symbol xedit::*syntax-symbols*)
(import symbol)
)
(in-package "XEDIT")
(makunbound '*syntax-symbols*)
#|
TODO:
o Add a command to match without increment the offset in the input, this
may be useful for example in a case like:
some-table
match "<"
switch -1
match "<" <- the table already eated this, so it won't be matched.
This must be carefully checked at compile time, such instruction should
be in a token that returns or starts a new one, and even then, may need
runtime check to make sure it won't enter an infinite loop.
o Allow combining properties, this is supported in Xaw, and could allow some
very interesting effects for complex documents.
o Maybe have an separated function/loop for tables that don't have tokens
that start/switch to another table, and/or have the contained attribute set.
This could allow running considerably faster.
o Do a better handling of interactive edition for tokens that start and end
with the same pattern, as an example strings, if the user types '"', it
will parse up to the end of the file, "inverting" all strings.
o Allow generic code to be run once a match is found, such code could handle
some defined variables and take decisions based on the parser state. This
should be detected at compile time, to maybe run a different parser for
such syntax tables, due to the extra time building the environment to
call the code. This would be useful to "really" parse documents with
complex syntax, for example, a man page source file.
o Add command to change current default property without initializing a new
state.
o Fix problems matching EOL. Since EOL is an empty string match, if there
is a rule to match only EOL, but some other rule matches up to the end
of the input, the match to EOL will not be recognized. Currently the only
way to handle this is to have a nested table that always returns once a
match is found, so that it will restart the match loop code even if the
input is at EOL.
One possible solution would be to add the ending newline to the input,
and then instead of matching "$", should match "\\n".
o XXX Usage of the variable newline-property must be reviewed in function
syntax-highlight, if the text property has a background attribute,
visual effect will look "strange", will paint a square with the
background attribute at the end of every line in the matched text.
|#
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Some annotations to later write documentation for the module...
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
#|
The current interface logic should be easy to understand for people
that have written lex scanners before. It has some extended semantics,
that could be translated to stacked BEGIN() statements in lex, but
currently does not have rules for matches in the format RE/TRAILING, as
well as code attached to rules (the biggest difference) and/or things
like REJECT and unput(). Also, at least currently, it is *really* quite
slower than lex.
MATCHING RULES
--------------
When two tokens are matched at the same input offset, the longest
token is used, if the length is the same, the first definition is
used. For example:
token1 => int
token2 => [A-Za-z]+
input => integer
Token1 matches "int" and token2 matches "integer", but since token2 is
longer, it is used. But in the case:
token1 => int
token2 => [A-Za-z]+
input => int
Both, token1 and token2 match "int", since token1 is defined first, it
is used.
|#
;; Initialize some default properties that may be shared in syntax
;; highlight definitions. Use of these default properties is encouraged,
;; so that "tokens" will be shown identically when editing program
;; sources in different programming languages.
(defsynprop *prop-default*
"default"
:font "*courier-medium-r*-12-*"
:foreground "black")
(defsynprop *prop-keyword*
"keyword"
:font "*courier-bold-r*-12-*"
:foreground "gray12")
(defsynprop *prop-number*
"number"
:font "*courier-bold-r*-12-*"
:foreground "OrangeRed3")
(defsynprop *prop-string*
"string"
:font "*lucidatypewriter-medium-r*-12-*"
:foreground "RoyalBlue2")
(defsynprop *prop-constant*
"constant"
:font "*lucidatypewriter-medium-r*-12-*"
:foreground "VioletRed3")
(defsynprop *prop-comment*
"comment"
:font "*courier-medium-o*-12-*"
:foreground "SlateBlue3")
(defsynprop *prop-preprocessor*
"preprocessor"
:font "*courier-medium-r*-12-*"
:foreground "green4")
(defsynprop *prop-punctuation*
"punctuation"
:font "*courier-bold-r*-12-*"
:foreground "gray12")
;; Control characters, not always errors...
(defsynprop *prop-control*
"control"
:font "*courier-bold-r*-12-*"
:foreground "yellow2"
:background "red3")
(defsynprop *prop-error*
"error"
:font "*new century schoolbook-bold*-24-*"
:foreground "yellow"
:background "red")
(defsynprop *prop-annotation*
"annotation"
:font "*courier-medium-r*-12-*"
:foreground "black"
:background "PaleGreen")
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; The "main" definition of the syntax highlight coding interface.
;; Creates a "special" variable with the given name, associating to
;; it an already compiled syntax table.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defmacro defsyntax (variable label property indent options &rest lists)
`(if (boundp ',variable)
,variable
(progn
(proclaim '(special ,variable))
(setq ,variable
(compile-syntax-table
(string ',variable) ,options
(syntable ,label ,property ,indent ,@lists)
)
)
)
)
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Just a wrapper to create a hash-table and bound it to a symbol.
;; Example of call:
;; (defsynoptions *my-syntax-options*
;; (:indent . 8)
;; (:indent-option-1 . 1)
;; (:indent-option-2 . 2)
;; )
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defmacro defsynoptions (variable &rest options)
`(if (boundp ',variable)
,variable
(progn
(proclaim '(special ,variable))
(setq ,variable (make-hash-table :initial-contents ',options))
)
)
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; These definitions should be "private".
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defstruct syntoken
regex ;; A compiled regexp.
property ;; NIL for default, or a synprop structure.
contained ;; Only used when switch/begin is not NIL. Values:
;; NIL -> just switch to or begin new
;; syntax table.
;; (not NIL) -> apply syntoken property
;; (or default one) to matched
;; text *after* switching to or
;; beginning a new syntax table.
switch ;; Values for switch are:
;; NIL -> do nothing
;; A keyword -> switch to the syntax table
;; identified by the keyword.
;; A negative integer -> Pop the stack
;; -<swich-value> times.
;; A common value is -1,
;; to switch to the previous
;; state, but some times
;; it is desired to return
;; two or more times in
;; in the stack.
;; NOTE: This is actually a jump, the stack is
;; popped until the named syntax table is found,
;; if the stack becomes empty, a new state is
;; implicitly created.
begin ;; NIL or a keyword (like switch), but instead of
;; popping the stack, it pushes the current syntax
;; table to the stack and sets a new current one.
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Just a wrapper to make-syntoken.
;; TODO: Add support for structure constructors.
;; XXX: Note that the NOSUB only works with the xedit regex, it
;; will still return the match offsets, but will ignore subexpressions,
;; that is, parenthesis are used only for grouping.
;; TODO: Create a new version of the re-exec call that returns
;; offsets in the format (<from> . <to>) and not
;; ((<from0> . <to0>) ... (<fromN> . <toN>)). Only the global result
;; is expected/used, so there is no reason to allocate more than one
;; cons cell per call.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defun syntoken (pattern
&key icase nospec property contained switch begin (nosub t)
&aux
(regex
(re-comp pattern :icase icase :nospec nospec :nosub nosub)
)
check)
;; Don't allow a regex that matches the null string enter the
;; syntax table list.
(if (consp (setq check (re-exec regex "" :noteol t :notbol t)))
#+xedit (error "SYNTOKEN: regex matches empty string ~S" regex)
#-xedit ()
)
(make-syntoken
:regex regex
:property property
:contained contained
:switch switch
:begin begin
)
)
;; This structure is defined only to do some type checking, it just
;; holds a list of keywords.
(defstruct synaugment
labels ;; List of keywords labeling syntax tables.
)
(defstruct syntable
label ;; A keyword naming this syntax table.
property ;; NIL or a default synprop structure.
indent ;; Indentation function for the syntax table.
tokens ;; A list of syntoken structures.
tables ;; A list of syntable structures.
augments ;; A list of synaugment structures, used only
;; at "compile time", so that a table can be
;; used before it's definition.
bol ;; One of the tokens match the empty string at
;; the start of a line (loop optimization hint).
;; Field filled at "link" time.
eol ;; Same comments as bol, but in this case, for
;; the empty string at the end of a line.
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Just call make-syntable, but sorts the elements by type, allowing
;; a cleaner code when defining the syntax highlight rules.
;; XXX Same comments as for syntoken about the use of a constructor for
;; structures. TODO: when/if clos is implemented in the interpreter.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defun syntable (label default-property indent &rest definitions)
;; Check for possible errors in the arguments.
(unless (keywordp label)
(error "SYNTABLE: ~A is not a keyword" label)
)
(unless
(or
(null default-property)
(synprop-p default-property)
)
(error "SYNTABLE: ~A is an invalid text property"
default-property
)
)
;; Don't allow unknown data in the definition list.
;; XXX typecase should be added to the interpreter, and since
;; the code is traversing the entire list, it could build
;; now the arguments to make-syntable.
(dolist (item definitions)
(unless
(or
;; Allow NIL in the definition list, so that one
;; can put conditionals in the syntax definition,
;; and if the conditional is false, fill the slot
;; with a NIL value.
(atom item)
(syntoken-p item)
(syntable-p item)
(synaugment-p item)
)
(error "SYNTABLE: invalid syntax table argument ~A" item)
)
)
;; Build the syntax table.
(make-syntable
:label label
:property default-property
:indent indent
:tokens (remove-if-not #'syntoken-p definitions)
:tables (remove-if-not #'syntable-p definitions)
:augments (remove-if-not #'synaugment-p definitions)
)
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Just to do a "preliminary" error checking, every element must be a
;; a keyword, and also check for reserved names.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defun synaugment (&rest keywords)
(dolist (keyword keywords)
(unless (keywordp keyword)
(error "SYNAUGMENT: bad syntax table label ~A" keyword)
)
)
(make-synaugment :labels keywords)
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Recursive compile utility function.
;; Returns a cons in the format:
;; car => List of all syntoken structures
;; (including child tables).
;; cdr => List of all child syntable structures.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defun list-syntable-elements (table &aux result sub-result)
(setq
result
(cons
(syntable-tokens table)
(syntable-tables table))
)
;; For every child syntax table.
(dolist (child (syntable-tables table))
;; Recursively call list-syntable-elements.
(setq sub-result (list-syntable-elements child))
(rplaca result (append (car result) (car sub-result)))
(rplacd result (append (cdr result) (cdr sub-result)))
)
;; Return the pair of nested tokens and tables.
result
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Append tokens of the augment list to the tokens of the specified
;; syntax table.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defun compile-syntax-augment-list (table table-list
&aux labels augment tokens)
;; Create a list of all augment tables.
(dolist (augment (syntable-augments table))
(setq labels (append labels (synaugment-labels augment)))
)
;; Remove duplicates and references to "itself",
;; without warnings?
(setq
labels
(remove
(syntable-label table)
(remove-duplicates labels :from-end t)
)
)
;; Check if the specified syntax tables exists!
(dolist (label labels)
(unless
(setq
augment
(car (member label table-list :key #'syntable-label))
)
(error "COMPILE-SYNTAX-AUGMENT-LIST: Cannot augment ~A in ~A"
label
(syntable-label table)
)
)
;; Increase list of tokens.
(setq tokens (append tokens (syntable-tokens augment)))
)
;; Store the tokens in the augment list. They will be added
;; to the syntax table in the second pass.
(setf (syntable-augments table) tokens)
;; Recurse on every child table.
(dolist (child (syntable-tables table))
(compile-syntax-augment-list child table-list)
)
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Just add the augmented tokens to the token list, recursing on
;; every child syntax table.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defun link-syntax-augment-table (table)
(setf
(syntable-tokens table)
;; When augmenting a table, duplicated tokens or different tokens
;; that use the same regex pattern should be common.
(remove-duplicates
(nconc (syntable-tokens table) (syntable-augments table))
:key #'syntoken-regex
:test #'equal
:from-end t
)
;; Don't need to keep this list anymore.
(syntable-augments table)
()
)
;; Check if one of the tokens match the empty string at the
;; start or end of a text line. XXX The fields bol and eol
;; are expected to be initialized to NIL.
(dolist (token (syntable-tokens table))
(when (consp (re-exec (syntoken-regex token) "" :noteol t))
(setf (syntable-bol table) t)
(return)
)
)
(dolist (token (syntable-tokens table))
(when (consp (re-exec (syntoken-regex token) "" :notbol t))
(setf (syntable-eol table) t)
(return)
)
)
(dolist (child (syntable-tables table))
(link-syntax-augment-table child)
)
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; "Compile" the main structure of the syntax highlight code.
;; Variables "switches" and "begins" are used only for error checking.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defun compile-syntax-table (name options main-table &aux syntax elements
switches begins tables properties)
(unless (stringp name)
(error "COMPILE-SYNTAX-TABLE: ~A is not a string" name)
)
(setq
elements
(list-syntable-elements main-table)
switches
(remove-if
#'null
(car elements)
:key #'syntoken-switch
)
begins
(remove-if-not
#'keywordp
(car elements)
:key #'syntoken-begin
)
;; The "main-table" isn't in the list, because
;; list-syntable-elements includes only the child tables;
;; this is done to avoid the need of removing duplicates here.
tables
(cons main-table (cdr elements))
)
;; Check for typos in the keywords, or for not defined syntax tables.
(dolist (item (mapcar #'syntoken-switch switches))
(unless
(or
(and
(integerp item)
(minusp item)
)
(member item tables :key #'syntable-label)
)
(error "COMPILE-SYNTAX-TABLE: SWITCH ~A cannot be matched"
item
)
)
)
(dolist (item (mapcar #'syntoken-begin begins))
(unless (member item tables :key #'syntable-label)
(error "COMPILE-SYNTAX-TABLE: BEGIN ~A cannot be matched"
item
)
)
)
;; Create a list of all properties used by the syntax.
(setq
properties
(delete-duplicates
;; Remove explicitly set to "default" properties.
(remove nil
(append
;; List all properties in the syntoken list.
(mapcar
#'syntoken-property
(car elements)
)
;; List all properties in the syntable list.
(mapcar
#'syntable-property
tables
)
)
)
:test #'string=
:key #'synprop-name
)
)
;; Provide a default property if none specified.
(unless
(member
"default"
properties
:test #'string=
:key #'synprop-name
)
(setq properties (append (list *prop-default*) properties))
)
;; Now that a list of all nested syntax tables is known, compile the
;; augment list. Note that even the main-table can be augmented to
;; include tokens of one of it's children.
;; Adding the tokens of the augment tables must be done in
;; two passes, or it may cause surprises due to "inherited"
;; tokens, as the augment table was processed first, and
;; increased it's token list.
(compile-syntax-augment-list main-table tables)
;; Now just append the augmented tokens to the table's token list.
(link-syntax-augment-table main-table)
;; Change all syntoken switch and begin fields to point to the
;; syntable.
(dolist (item switches)
(if (keywordp (syntoken-switch item))
;; A switch may be relative, check if a keyword
;; was specified.
(setf
(syntoken-switch item)
(car
(member
(syntoken-switch item)
tables
:key #'syntable-label
)
)
)
)
)
(dolist (item begins)
(setf
(syntoken-begin item)
(car
(member
(syntoken-begin item)
tables
:key #'syntable-label
)
)
)
)
;; Don't need to add a entity for default properties
(dolist (item (car elements))
(and
(syntoken-property item)
(string= (synprop-name (syntoken-property item)) "default")
(setf (syntoken-property item) ())
)
)
(dolist (item tables)
(and
(syntable-property item)
(string= (synprop-name (syntable-property item)) "default")
(setf (syntable-property item) ())
)
)
(setq syntax
(make-syntax
:name name
:options options
:labels tables
:quark
(compile-syntax-property-list
name
properties
)
:token-count
(length (car elements))
)
)
;; Ready to run!
)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Loop applying the specifed syntax table to the text.
;; XXX This function needs a review. Should compile the regex patterns
;; with newline sensitive match (and scan the entire file), and keep a
;; cache of matched tokens (that may be at a very longer offset), and,
;; when the match is removed from the cache, readd the token to the
;; token-list; if the token does not match, it will not be in the cache,
;; but should be removed from the token-list. If properly implemented, it
;; should be somewhat like 4 times faster, but I would not be surprised
;; if it becames even faster.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defun syntax-highlight (*syntax*
&optional
(*from* (point-min))
(*to* (point-max))
interactive
&aux
#+debug (*line-number* 0)
stream
indent-table
)
;; Make sure the property list is in use.
;; The interactive flag is only set after loading the file.
(or interactive
(property-list (syntax-quark *syntax*))
)
#+debug
(setq *from* 0 *to* 0)
#-debug
(and (>= *from* *to*) (return-from syntax-highlight (values *from* nil)))
;; Remove any existing properties from the text.
(clear-entities *from* (1+ *to*))
(setq stream
#-debug (make-string-input-stream (read-text *from* (- *to* *from*)))
#+debug *standard-input*
)
(prog*
(
;; Used to check if end of file found but syntax stack did
;; not finish.
(point-max (point-max))
;; Used in interactive mode, to return the syntax table
;; where the cursor is located.
(point (point))
;; The current stack of states.
stack
;; The current syntable.
(syntax-table (car (syntax-labels *syntax*)))
;; The current syntable's default property.
(default-property (syntable-property syntax-table))
;; Add this property to newlines as a hint to the interactive
;; callback, so that it knows from where to restart parsing.
newline-property
;; The tokens in the current syntax table that may match,
;; i.e. the items in this list are not in nomatch.
token-list
;; A pointer to the syntable token list, if token-list is
;; eq to this value, cannot change it inplace.
current-token-list
;; Help to avoid allocating too many new object cells, and
;; optmizes a bit time in [n]?set-difference.
;; This optimizes only the processing of one line of text
;; as nomatch must be rebuilt when reading a new line of text.
token-list-stack
;; Matches for the current list of tokens.
matches
;; Line of text.
line
;; Length of the text line.
length
;; A inverse cache, don't call re-exec when the regex is
;; already known to not match.
nomatch
;; Use cache as a list of matches to avoid repetitive
;; unnecessary calls to re-exec.
;; cache is a list in which every element has the format:
;; (token . (start . end))
;; Line of text.
cache
;; Used just to avoid a function call at every re-exec call.
notbol
match
start
left
right
result
property
;; Beginig a new syntax table?
begin
;; Switching to another syntax table?
switch
;; Property flag when changing the current syntax table.
contained
;; Flag to know if syntax table has changed.
change
;; Variables used when removing invalid elements from the
;; the cache.
item
from
to
)
;-----------------------------------------------------------------------
:read
#+debug-verbose
(format t "** Entering :READ stack length is ~D~%" (length stack))
#+debug (format t "~%[~D]> " (incf *line-number*))
;; If input has finished, return.
(unless (setq line (read-line stream nil nil))
(when
(and
;; If a nested syntax table wasn't finished
(consp stack)
(<
(setq *to* (scan *from* :eol :right))
point-max
)
)
(setq line (read-text *from* (- *to* *from*)))
(clear-entities *from* (1+ *to*))
(go :again)
)
#-debug (close stream)
(return)
)
;------------------------------------------------------------------------
:again
(setq
start 0
length (length line)
token-list (syntable-tokens syntax-table)
current-token-list token-list
token-list-stack ()
nomatch ()
cache ()
)
;; If empty line, and current table does not have matches for
;; the empty string at start or end of a text line.
(when
(and
(= length 0)
(not (syntable-eol syntax-table))
(not (syntable-bol syntax-table)))
#+debug-verbose
(format t "Empty line and table has no match to bol or eol~%")
(and newline-property
(add-entity *from* 1 (synprop-quark newline-property)))
(go :update)
)
;------------------------------------------------------------------------
:loop
#+debug-verbose
(format t "** Entering :LOOP at offset ~D in table ~A, cache has ~D items~%"
start
(syntable-label syntax-table)
(length cache))
(setq notbol (> start 0))
;; For every token that may match.
(dolist
(token
(setq
token-list
(if (eq token-list current-token-list)
(set-difference token-list nomatch :test #'eq)
(nset-difference token-list nomatch :test #'eq)
)
)
)
;; Try to fetch match from cache.
(if (setq match (member token cache :test #'eq :key #'car))
;; Match is in the cache.
(progn
;; Match must be moved to the beginning of the
;; matches list, as a match from another syntax
;; table may be also in the cache, but before
;; the match for the current token.
#+debug-verbose (format t "cached: {~A:~S} ~A~%"
(cdar match)
(subseq line (cadar match) (cddar match))
(syntoken-regex token))
;; Remove the match from the cache.
(if (eq match cache)
;; This could be changed to only set "matches"
;; if it is not the first element of cache,
;; but is unsafe, because other tokens may
;; be added to "matches", and will end up
;; before when joining "matches" and "cache".
(progn
(setq cache (cdr cache))
(rplacd match matches)
(setq matches match))
(progn
(if (= (length match) 1)
(progn
(rplacd (last cache 2) nil)
(rplacd match matches)
(setq matches match))
(progn
(setq matches (cons (car match) matches))
(rplaca match (cadr match))
(rplacd match (cddr match)))
)
)
)
;; Exit loop if the all the remaining
;; input was matched.
(when
(and
(= start (cadar match))
(= length (cddar match))
)
#+debug-verbose (format t "Rest of line match~%")
(return)
)
)
;; Not in the cache, call re-exec.
(if
(consp
(setq
match
(re-exec
(syntoken-regex token)
line
:start start
:notbol notbol)))
;; Match found.
(progn
#+debug-verbose (format t "Adding to cache: {~A:~S} ~A~%"
(car match)
(subseq line (caar match) (cdar match))
(syntoken-regex token))
;; Only the first pair is used.
(setq match (car match))
(cond
(
(or
(null matches)
;; No overlap and after most
;; recent match.
(>= (car match) (cddar matches))
;; No overlap and before most
;; recent match.
(<= (cdr match) (cadar matches))
)
(setq
matches
(cons (cons token match) matches)
)
)
(
(or
;; Overlap, but start before most
;; recent match.
(< (car match) (cadar matches))
(and
;; Same offset as most recent
;; match, but is longer.
(= (car match) (cadar matches))
(> (cdr match) (cddar matches))
)
)
(rplaca (car matches) token)
(rplacd (car matches) match)
#+debug-verbose (format t "Replaced most recent match~%")
)
(t
#+debug-verbose (format t "Ignored~%")
;; XXX The interpreter does not yet implement
;; implicit tagbody in dolist, just comment
;; the go call in that case. (Will just do
;; an unecessary test...)
(go :ignored)
)
)
;; Exit loop if the all the remaining
;; input was matched.
(when
(and
(= start (car match))
(= length (cdr match)))
#+debug-verbose (format t "Rest of line match~%")
(return))
)
;; Match not found.
(progn
#+debug-verbose (format t "Adding to nomatch: ~A~%"
(syntoken-regex token))
(setq nomatch (cons token nomatch)))
)
)
:ignored
)
;; Add matches to the beginning of the cache list.
(setq
;; Put matches with smaller offset first.
cache
(stable-sort (nconc (nreverse matches) cache) #'< :key #'cadr)
;; Make sure that when the match loop is reentered, this
;; variable is NIL.
matches
()
)
;; While the first entry in the cache is not from the current table.
(until (or (null cache) (member (caar cache) token-list :test #'eq))
#+debug-verbose
(format t "Not in the current table, removing {~A:~S} ~A~%"
(cdar cache)
(subseq line (cadar cache) (cddar cache))
(syntoken-regex (caar cache)))
(setq cache (cdr cache))
)
;; If nothing was matched in the entire/remaining line.
(unless cache
(when default-property
(if
(or
(null result)
(> start (cadar result))
(not (eq (cddar result) default-property)))
(setq
result
(cons
(cons start (cons length default-property))
result
)
)
(rplaca (cdar result) length)
)
)
#+debug-verbose
(format t "No match until end of line~%")
;; Result already known, and there is no syntax table
;; change, bypass :PARSE.
(and interactive
(null indent-table)
(<= 0 (- point *from*) length)
(setq indent-table syntax-table))
(go :process)
)
#+debug-verbose
(format t "Removing first candidate from cache {~A:~S} ~A~%"
(cdar cache)
(subseq line (cadar cache) (cddar cache))
(syntoken-regex (caar cache))
)
;; Prepare to choose best match.
(setq
match (car cache)
left (cadr match)
right (cddr match)
cache (cdr cache)
)
;; First element can be safely removed now.
;; If there is only one, skip loop below.
(or cache (go :parse))
;; Remove elements of cache that must be discarded.
(setq
item (car cache)
from (cadr item)
to (cddr item)
)
(loop
(if
(or
;; If everything removed from the cache.
(null item)
;; Or next item is at a longer offset than the
;; end of current match.
(>= from right)
)
(return)
)
(and
;; If another match at the same offset.
(= left from)
;; And if this match is longer than the current one.
(> to right)
(member (car item) token-list :test #'eq)
(setq
match item
right to
)
)
#+debug-verbose
(format t "Removing from cache {~A:~S} ~A~%"
(cdar cache)
(subseq line from to)
(syntoken-regex (caar cache)))
(setq
cache (cdr cache)
item (car cache)
from (cadr item)
to (cddr item)
)
)
;-----------------------------------------------------------------------
:parse
#+debug-verbose
(format t "** Entering :PARSE~%")
(setq
;; Change match value to the syntoken.
match (car match)
begin (syntoken-begin match)
switch (syntoken-switch match)
contained (syntoken-contained match)
change (or begin switch)
)
;; Check for unmatched leading text.
(when (and default-property (> left start))
#+debug-verbose (format t "No match in {(~D . ~D):~S}~%"
start
left
(subseq line start left)
)
(if
(or
(null result)
(> start (cadar result))
(not (eq (cddar result) default-property)))
(setq
result
(cons
(cons start (cons left default-property))
result
)
)
(rplaca (cdar result) left)
)
)
;; If the syntax table is not changed,
;; or if the new table requires that the
;; current default property be used.
(unless (and change contained)
(and
(> right left)
(setq
property
(or
;; If token specifies the property.
(syntoken-property match)
default-property
)
)
;; Add matched text.
(if
(or
(null result)
(> left (cadar result))
(not (eq (cddar result) property))
)
(setq
result
(cons
(cons left (cons right property))
result
)
)
(rplaca (cdar result) right)
)
)
#+debug-verbose
(format t "(0)Match found for {(~D . ~D):~S}~%"
left
right
(subseq line left right)
)
)
;; Update start offset in the input now!
(and interactive
(null indent-table)
(<= start (- point *from*) right)
(setq indent-table syntax-table))
(setq start right)
;; When changing the current syntax table.
(when change
(when switch
(if (numberp switch)
;; If returning to a previous state.
;; Don't generate an error if the stack
;; becomes empty?
(while
(< switch 0)
(setq
syntax-table (pop stack)
token-list (pop token-list-stack)
switch (1+ switch)
)
)
;; Else, not to a previous state, but
;; returning to a named syntax table,
;; search for it in the stack.
(while
(and
(setq
token-list (pop token-list-stack)
syntax-table (pop stack)
)
(not (eq switch syntax-table))
)
;; Empty loop.
)
)
;; If no match found while popping
;; the stack.
(unless syntax-table
;; Return to the topmost syntax table.
(setq
syntax-table
(car (syntax-labels *syntax*))
)
)
#+debug-verbose (format t "switching to ~A offset: ~D~%"
(syntable-label syntax-table)
start
)
(if (null token-list)
(setq token-list (syntable-tokens syntax-table))
)
)
(when begin
;; Save state for a possible
;; :SWITCH later.
(setq
stack (cons syntax-table stack)
token-list-stack (cons token-list token-list-stack)
token-list (syntable-tokens begin)
syntax-table begin
)
#+debug-verbose (format t "begining ~A offset: ~D~%"
(syntable-label syntax-table)
start
)
)
;; Change current syntax table.
(setq
default-property (syntable-property syntax-table)
current-token-list (syntable-tokens syntax-table)
)
;; Set newline property, to help interactive callback
;; Only need to have a defined value, for now don't care
;; about wich value is being used, neither if there is
;; a value to be set.
(if (null stack)
(setq newline-property nil)
(or newline-property
(setq newline-property default-property)
(setq newline-property (syntoken-property match))
)
)
;; If processing of text was deferred.
(when contained
(and
(> right left)
(setq
property
(or
(syntoken-property match)
default-property
)
)
;; Add matched text with the updated property.
(if
(or
(null result)
(> left (cadar result))
(not (eq (cddar result) property))
)
(setq
result
(cons
(cons left (cons right property))
result
)
)
(rplaca (cdar result) right)
)
)
#+debug-verbose (format t "(1)Match found for {(~D . ~D):~S}~%"
left
right
(subseq line left right)
)
)
(go :loop)
)
;-----------------------------------------------------------------------
;; Wait for the end of the line to process, so that
;; it is possible to join sequential matches with the
;; same text property.
(and (or cache (< start length)) (go :loop))
:process
#+debug-verbose
(format t "** Entering :PROCESS~%")
(if result
(progn
;; If the last property was at the end of the line,
;; there are nested syntax tables, and there is a
;; default property, include the newline in the property,
;; as a hint to the interactive callback.
(and
newline-property
(if
(and
(eq (cddar result) newline-property)
(= length (cadar result))
)
(rplaca (cdar result) (1+ length))
(setq
result
(cons
(cons length (cons (1+ length) newline-property))
result
)
)
)
)
;; Result was created in reversed order.
(nreverse result)
(dolist (item result)
(setq
left (car item)
right (cadr item)
property (cddr item))
;; Use the information.
(add-entity
(+ *from* left)
(- right left)
(synprop-quark property))
)
)
(and newline-property
(add-entity
(+ *from* length)
1
(synprop-quark newline-property))
)
)
;------------------------------------------------------------------------
:update
;; Prepare for new matches.
(setq
result nil
;; Update offset to read text.
;; Add 1 for the skipped newline.
*from* (+ *from* length 1)
)
(go :read)
)
#+debug (terpri)
(values *to* indent-table)
)
(compile 'syntax-highlight)
|