/usr/include/clang/Lex/Token.h is in libclang-dev 3.0-6ubuntu3.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 | //===--- Token.h - Token interface ------------------------------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file defines the Token interface.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_TOKEN_H
#define LLVM_CLANG_TOKEN_H
#include "clang/Basic/TemplateKinds.h"
#include "clang/Basic/TokenKinds.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/OperatorKinds.h"
#include <cstdlib>
namespace clang {
class IdentifierInfo;
/// Token - This structure provides full information about a lexed token.
/// It is not intended to be space efficient, it is intended to return as much
/// information as possible about each returned token. This is expected to be
/// compressed into a smaller form if memory footprint is important.
///
/// The parser can create a special "annotation token" representing a stream of
/// tokens that were parsed and semantically resolved, e.g.: "foo::MyClass<int>"
/// can be represented by a single typename annotation token that carries
/// information about the SourceRange of the tokens and the type object.
class Token {
/// The location of the token.
SourceLocation Loc;
// Conceptually these next two fields could be in a union. However, this
// causes gcc 4.2 to pessimize LexTokenInternal, a very performance critical
// routine. Keeping as separate members with casts until a more beautiful fix
// presents itself.
/// UintData - This holds either the length of the token text, when
/// a normal token, or the end of the SourceRange when an annotation
/// token.
unsigned UintData;
/// PtrData - This is a union of four different pointer types, which depends
/// on what type of token this is:
/// Identifiers, keywords, etc:
/// This is an IdentifierInfo*, which contains the uniqued identifier
/// spelling.
/// Literals: isLiteral() returns true.
/// This is a pointer to the start of the token in a text buffer, which
/// may be dirty (have trigraphs / escaped newlines).
/// Annotations (resolved type names, C++ scopes, etc): isAnnotation().
/// This is a pointer to sema-specific data for the annotation token.
/// Other:
/// This is null.
void *PtrData;
/// Kind - The actual flavor of token this is.
///
unsigned short Kind;
/// Flags - Bits we track about this token, members of the TokenFlags enum.
unsigned char Flags;
public:
// Various flags set per token:
enum TokenFlags {
StartOfLine = 0x01, // At start of line or only after whitespace.
LeadingSpace = 0x02, // Whitespace exists before this token.
DisableExpand = 0x04, // This identifier may never be macro expanded.
NeedsCleaning = 0x08, // Contained an escaped newline or trigraph.
LeadingEmptyMacro = 0x10 // Empty macro exists before this token.
};
tok::TokenKind getKind() const { return (tok::TokenKind)Kind; }
void setKind(tok::TokenKind K) { Kind = K; }
/// is/isNot - Predicates to check if this token is a specific kind, as in
/// "if (Tok.is(tok::l_brace)) {...}".
bool is(tok::TokenKind K) const { return Kind == (unsigned) K; }
bool isNot(tok::TokenKind K) const { return Kind != (unsigned) K; }
/// isAnyIdentifier - Return true if this is a raw identifier (when lexing
/// in raw mode) or a non-keyword identifier (when lexing in non-raw mode).
bool isAnyIdentifier() const {
return is(tok::identifier) || is(tok::raw_identifier);
}
/// isLiteral - Return true if this is a "literal", like a numeric
/// constant, string, etc.
bool isLiteral() const {
return is(tok::numeric_constant) || is(tok::char_constant) ||
is(tok::wide_char_constant) || is(tok::utf16_char_constant) ||
is(tok::utf32_char_constant) || is(tok::string_literal) ||
is(tok::wide_string_literal) || is(tok::utf8_string_literal) ||
is(tok::utf16_string_literal) || is(tok::utf32_string_literal) ||
is(tok::angle_string_literal);
}
bool isAnnotation() const {
#define ANNOTATION(NAME) \
if (is(tok::annot_##NAME)) \
return true;
#include "clang/Basic/TokenKinds.def"
return false;
}
/// getLocation - Return a source location identifier for the specified
/// offset in the current file.
SourceLocation getLocation() const { return Loc; }
unsigned getLength() const {
assert(!isAnnotation() && "Annotation tokens have no length field");
return UintData;
}
void setLocation(SourceLocation L) { Loc = L; }
void setLength(unsigned Len) {
assert(!isAnnotation() && "Annotation tokens have no length field");
UintData = Len;
}
SourceLocation getAnnotationEndLoc() const {
assert(isAnnotation() && "Used AnnotEndLocID on non-annotation token");
return SourceLocation::getFromRawEncoding(UintData);
}
void setAnnotationEndLoc(SourceLocation L) {
assert(isAnnotation() && "Used AnnotEndLocID on non-annotation token");
UintData = L.getRawEncoding();
}
SourceLocation getLastLoc() const {
return isAnnotation() ? getAnnotationEndLoc() : getLocation();
}
/// getAnnotationRange - SourceRange of the group of tokens that this
/// annotation token represents.
SourceRange getAnnotationRange() const {
return SourceRange(getLocation(), getAnnotationEndLoc());
}
void setAnnotationRange(SourceRange R) {
setLocation(R.getBegin());
setAnnotationEndLoc(R.getEnd());
}
const char *getName() const {
return tok::getTokenName( (tok::TokenKind) Kind);
}
/// startToken - Reset all flags to cleared.
///
void startToken() {
Kind = tok::unknown;
Flags = 0;
PtrData = 0;
UintData = 0;
Loc = SourceLocation();
}
IdentifierInfo *getIdentifierInfo() const {
assert(isNot(tok::raw_identifier) &&
"getIdentifierInfo() on a tok::raw_identifier token!");
assert(!isAnnotation() &&
"getIdentifierInfo() on an annotation token!");
if (isLiteral()) return 0;
return (IdentifierInfo*) PtrData;
}
void setIdentifierInfo(IdentifierInfo *II) {
PtrData = (void*) II;
}
/// getRawIdentifierData - For a raw identifier token (i.e., an identifier
/// lexed in raw mode), returns a pointer to the start of it in the text
/// buffer if known, null otherwise.
const char *getRawIdentifierData() const {
assert(is(tok::raw_identifier));
return reinterpret_cast<const char*>(PtrData);
}
void setRawIdentifierData(const char *Ptr) {
assert(is(tok::raw_identifier));
PtrData = const_cast<char*>(Ptr);
}
/// getLiteralData - For a literal token (numeric constant, string, etc), this
/// returns a pointer to the start of it in the text buffer if known, null
/// otherwise.
const char *getLiteralData() const {
assert(isLiteral() && "Cannot get literal data of non-literal");
return reinterpret_cast<const char*>(PtrData);
}
void setLiteralData(const char *Ptr) {
assert(isLiteral() && "Cannot set literal data of non-literal");
PtrData = const_cast<char*>(Ptr);
}
void *getAnnotationValue() const {
assert(isAnnotation() && "Used AnnotVal on non-annotation token");
return PtrData;
}
void setAnnotationValue(void *val) {
assert(isAnnotation() && "Used AnnotVal on non-annotation token");
PtrData = val;
}
/// setFlag - Set the specified flag.
void setFlag(TokenFlags Flag) {
Flags |= Flag;
}
/// clearFlag - Unset the specified flag.
void clearFlag(TokenFlags Flag) {
Flags &= ~Flag;
}
/// getFlags - Return the internal represtation of the flags.
/// Only intended for low-level operations such as writing tokens to
// disk.
unsigned getFlags() const {
return Flags;
}
/// setFlagValue - Set a flag to either true or false.
void setFlagValue(TokenFlags Flag, bool Val) {
if (Val)
setFlag(Flag);
else
clearFlag(Flag);
}
/// isAtStartOfLine - Return true if this token is at the start of a line.
///
bool isAtStartOfLine() const { return (Flags & StartOfLine) ? true : false; }
/// hasLeadingSpace - Return true if this token has whitespace before it.
///
bool hasLeadingSpace() const { return (Flags & LeadingSpace) ? true : false; }
/// isExpandDisabled - Return true if this identifier token should never
/// be expanded in the future, due to C99 6.10.3.4p2.
bool isExpandDisabled() const {
return (Flags & DisableExpand) ? true : false;
}
/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier.
bool isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const;
/// getObjCKeywordID - Return the ObjC keyword kind.
tok::ObjCKeywordKind getObjCKeywordID() const;
/// needsCleaning - Return true if this token has trigraphs or escaped
/// newlines in it.
///
bool needsCleaning() const { return (Flags & NeedsCleaning) ? true : false; }
/// \brief Return true if this token has an empty macro before it.
///
bool hasLeadingEmptyMacro() const {
return (Flags & LeadingEmptyMacro) ? true : false;
}
};
/// PPConditionalInfo - Information about the conditional stack (#if directives)
/// currently active.
struct PPConditionalInfo {
/// IfLoc - Location where the conditional started.
///
SourceLocation IfLoc;
/// WasSkipping - True if this was contained in a skipping directive, e.g.
/// in a "#if 0" block.
bool WasSkipping;
/// FoundNonSkip - True if we have emitted tokens already, and now we're in
/// an #else block or something. Only useful in Skipping blocks.
bool FoundNonSkip;
/// FoundElse - True if we've seen a #else in this block. If so,
/// #elif/#else directives are not allowed.
bool FoundElse;
};
} // end namespace clang
namespace llvm {
template <>
struct isPodLike<clang::Token> { static const bool value = true; };
} // end namespace llvm
#endif
|