/usr/share/ucto/tokconfig-eng

version=0.2
[RULE-ORDER]
SUFFIX WORD-TOKEN ABBREVIATION-KNOWN NUMBER-ORDINAL URL URL-WWW URL-DOMAIN
E-MAIL WORD-PARPREFIX WORD-PARSUFFIX WORD-COMPOUND
ABBREVIATION INITIAL SMILEY REVERSE-SMILEY PUNCTUATION-MULTI DATE-REVERSE DATE
NUMBER-YEAR TIME FRACNUMBER NUMBER CURRENCY WORD PUNCTUATION UNKNOWN

[META-RULES]
SPLITTER=%
NUMBER-ORDINAL = \p{N}+-?(?: %ORDINALS% )(?:\Z|\P{Lu}|\P{Ll})$
#ABBREVIATION-KNOWN = (?:\p{P}*)?(?:\A|[^\p{L}\.])((?:%ABBREVIATIONS%)\.)(?:\Z|\P{L})
ABBREVIATION-KNOWN = (?:\p{P}*)?(?:\A|[^\p{L}\.])(?:\A)((?:%ABBREVIATIONS%)(?:\.{0,1}))(?:\Z|\P{L})
WORD-TOKEN =(%TOKENS%)(?:\p{P}*)?$
#PREFIX = (?:\A|[^\p{Lu}\.]|[^\p{Ll}\.])(%PREFIXES% )(?:\p{L}+)
SUFFIX = (?:\A|\p{L})+( %SUFFIXES% )(?:\Z|\P{L})

[RULES]
%include url
%include e-mail
%include smiley

#Ex: (dis)information
WORD-PARPREFIX=(?:\p{Ps}\p{L}+[\p{Pc}\p{Pd}]?\p{Pe}[\p{Pc}\p{Pd}]?)\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*

#Ex: understand(s)
WORD-PARSUFFIX=\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)*(?:[\p{Pc}\p{Pd}]?\p{Ps}[\p{Pc}\p{Pd}]?\p{L}+\p{Pe})

#Keep dash/underscore connected parts (even if they are in parenthesis)
WORD-COMPOUND=\p{L}+(?:[\p{Pc}\p{Pd}]\p{L}+)+

#Abbreviations with multiple periods
ABBREVIATION=^(\p{L}{1,3}(?:\.\p{L}{1,3})+\.?)(?:\Z|[,:;])

#retain initials
INITIAL=^(?:\p{Lt}|\p{Lu})\.$

#Homogeneous punctuation (ellipsis etc)
PUNCTUATION-MULTI=(?:\.|\-|[!\?]){2,}

#Date
DATE=\p{N}{1,2}/\p{Ps}?\p{N}{1,2}[/]\p{Ps}?\p{N}{2,4}
DATE-REVERSE=\p{N}{4}/\p{N}{1,2}/\p{N}{1,2}

FRACNUMBER=\p{N}+(?:/\p{N}+)+

NUMBER-YEAR=('\p{N}{2})(?:\P{N}|\z)
#NUMBER-YEAR=('\p{N}{2})\P{N}

#Times
TIME=\p{N}{1,2}:\p{N}{1,2}(?::\p{N})?(?i:a\.?m\.?|p\.?m\.?)?

#retain digits, including those starting with initial period (.22), and negative numbers
NUMBER=-?(?:[\.,]?\p{N}+)+

CURRENCY=\p{Sc}

WORD=[\p{L}\p{Mn}]+

PUNCTUATION=\p{P}

UNKNOWN=.

[PREFIXES]

[SUFFIXES]
n't|N'T
're|'RE
'm|'M
's|'S
'd|'D
've|'VE
'll|'LL

[ORDINALS]
st|ST
nd|ND
rd|RD
th|TH

[TOKENS]
't|'T
n/a|N/A

[UNITS]
km
m
cm
mm
g
kg
C
l
s
sec
min
gb
mb
kb


[CURRENCY]
USD
GBP
CAD
NZD
AUD
SGD
HKD
EUR

[ABBREVIATIONS]
acc
AD
Adm
al
Ala
anon
Apr
Ariz
Ark
arr
assoc
Aug
av
Ave
Bancorp
Bart
BC
Bhd
Bros
B.S
B.Sc
Calif
cap
Capt
cf
Cie
Co
CO
col
Col
Colo
comb
comb.form
compar
Conn
cont
contd
contr
Corp
CORP
Cos
COS
cu
Dec
Del
dept
Dept
dist
div
D-Mass
doc
doz
Dr
e.g
esp
Esq
est
etc
Etc
Ex
Feb
fem
ff
fig
Fla
for
Fri
ft
Ga
Gen
gm
Gov
Hon
Ill
Inc
INC
Ind
inst
Jan
Jansz
Jos
Jr
Jul
Jun
Kan
Ky
La
Lt
Ltd
M\.A
M\.Sc
MA
MSc
Maj
masc
Mass
Md
Messrs
met
Mfg
Mich
Minn
Miss
Mo
Mon
Mr
Mrs
Ms
Neb
neg
Nev
no
No
nom
Nos
Nov
Oct
Okla
Ore
Pa
pass
pers
Ph
phr
pla
poss
pres
Prof
Prop
Pty
ref
refl
Rep
Reps
Rev
sc
Sen
Sens
Sept
sing
Sr
St
superl
Tenn
Tex
Tues
usu
v
Va
var
viz
vs
Vt
Wash
Wis
Wyo


[FILTER]
ﬂ fl
ﬀ ff
ﬃ ffi
ﬄ ffl
# also filter soft hyphen
\u00AD


[EOSMARKERS]
%include standard-eos

[QUOTES]
%include standard-quotes
uctodata 0.4-1 / usr / share / ucto / tokconfig-eng
