blob: 1f45f8671c44faf85ee837b8bd52eab7db7f7620 [file] [log] [blame]
Marc Slemkoc0e07a22006-08-09 23:34:57 +00001#!python
Marc Slemkob2039e72006-08-09 01:00:17 +00002""" Thrift IDL parser/compiler
3
4 This parser uses the Python PLY LALR parser generator to build a parser for the Thrift IDL grammar.
5
6 If a compiles \"thyc\" file exists for a given source \"thrift\" file it computes a hash of the file and determines
7 if if it is the source of the \"thyc\" file. If so, it simply returns the parse tree previously computed, otherwise it
8 parses the source and generates a new \"thyc\" file (assuming of course the source file contains no errors.)
9
10 When the parser encounters import statements it searches for corresponding \"thrift\" or \"thyc\" files in paths corresponding to
11 the specified namespace.
12
13 Author(s): Mark Slee(mclee@facebook.com), Marc Kwiatkowski (marc@facebook.com)
14
15 $Id:
16"""
17
18import lex
19import os
20import pickle
21import string
Marc Slemkob2039e72006-08-09 01:00:17 +000022import yacc
23
24class Error(object):
25
26 def __init__(self, start=0, end=0, message=""):
27 if len(message) == 0:
28 raise Exception, "NO MESSAGE"
29 self.message = message
30 self.start = start
31 self.end = end
32
33 def __str__(self):
34 return str(self.start)+": error: "+self.message
35
36class SyntaxError(Error):
Marc Slemko27340eb2006-08-10 20:45:55 +000037 def __init__(self, yaccSymbol):
38 if isinstance(yaccSymbol, yacc.YaccSymbol):
39 Error.__init__(self, yaccSymbol.lineno, yaccSymbol.lineno, "syntax error "+str(yaccSymbol.value))
40 else:
41 Error.__init__(self, 1, 1, "syntax error "+str(yaccSymbol))
Marc Slemkob2039e72006-08-09 01:00:17 +000042
43class SymanticsError(Error):
44
45 def __init__(self, definition, message):
46 Error.__init__(self, definition.start, definition.end, message)
47 self.definition = definition
48
49 def __str__(self):
50 return str(self.start)+": error: "+self.message
51
52class ErrorException(Exception):
53
54 def __init__(self, errors=None):
55 self.errors = errors
56
57class Definition(object):
58 """ Abstract thrift IDL definition unit """
59
60 def __init__(self, symbols=None, name="", id=None):
61 if symbols:
62 self.lines(symbols)
63 self.name = name
64 self.id = id
65
66 def validate(self):
67 pass
68
69 def lines(self, symbols):
70 self.start = symbols.lineno(1)
71 self.end = symbols.lineno(len(symbols) - 1)
72
73class Identifier(Definition):
74 """ An Identifier - name and optional integer id """
75
76 def __init__(self, symbols, name, id=None):
77 Definition.__init__(self, symbols, name, id)
78
79 def __str__(self):
80 result = self.name
81 if self.id != 0:
82 result+="="+str(self.id)
83 return result
84
Marc Slemko27340eb2006-08-10 20:45:55 +000085
86def toCanonicalType(ttype):
87 if isinstance(ttype, TypeDef):
88 return toCanonicalType(ttype.definitionType)
89 else:
90 return ttype
91
92def isComparableType(ttype):
93 ttype = toCanonicalType(ttype)
94 return isinstance(ttype, PrimitiveType) or isinstance(ttype, Enum)
95
Marc Slemkob2039e72006-08-09 01:00:17 +000096class Type(Definition):
97 """ Abstract Type definition """
98
99 def __init__(self, symbols, name):
100 Definition.__init__(self, symbols, name)
101 self.name = name
102
103 def __str__(self):
104 return self.name
105
106class TypeDef(Type):
107
108 def __init__(self, symbols, name, definitionType):
109 Type.__init__(self, symbols, name)
110 self.definitionType = definitionType
111
112 def __str__(self):
113 return self.name+"<"+str(self.name)+", "+str(self.definitionType)+">"
114
115""" Primitive Types """
116
117class PrimitiveType(Type):
118
119 def __init__(self, name):
120 Type.__init__(self, None, name)
121
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000122STOP_TYPE = PrimitiveType("stop")
Marc Slemkob2039e72006-08-09 01:00:17 +0000123VOID_TYPE = PrimitiveType("void")
124BOOL_TYPE = PrimitiveType("bool")
125STRING_TYPE =PrimitiveType("utf7")
126UTF7_TYPE = PrimitiveType("utf7")
127UTF8_TYPE = PrimitiveType("utf8")
128UTF16_TYPE = PrimitiveType("utf16")
129BYTE_TYPE = PrimitiveType("u08")
130I08_TYPE = PrimitiveType("i08")
131I16_TYPE = PrimitiveType("i16")
132I32_TYPE = PrimitiveType("i32")
133I64_TYPE = PrimitiveType("i64")
134U08_TYPE = PrimitiveType("u08")
135U16_TYPE = PrimitiveType("u16")
136U32_TYPE = PrimitiveType("u32")
137U64_TYPE = PrimitiveType("u64")
138FLOAT_TYPE = PrimitiveType("float")
139
140PRIMITIVE_MAP = {
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000141 "stop" : STOP_TYPE,
Marc Slemkob2039e72006-08-09 01:00:17 +0000142 "void" : VOID_TYPE,
143 "bool" : BOOL_TYPE,
144 "string": UTF7_TYPE,
145 "utf7": UTF7_TYPE,
146 "utf8": UTF8_TYPE,
147 "utf16": UTF16_TYPE,
148 "byte" : U08_TYPE,
149 "i08": I08_TYPE,
150 "i16": I16_TYPE,
151 "i32": I32_TYPE,
152 "i64": I64_TYPE,
153 "u08": U08_TYPE,
154 "u16": U16_TYPE,
155 "u32": U32_TYPE,
156 "u64": U64_TYPE,
157 "float": FLOAT_TYPE
158}
159
160""" Collection Types """
161
162class CollectionType(Type):
163
164 def __init__(self, symbols, name):
165 Type.__init__(self, symbols, name)
166
Marc Slemko27340eb2006-08-10 20:45:55 +0000167 def validate(self):
168 return True
169
Marc Slemkob2039e72006-08-09 01:00:17 +0000170class Map(CollectionType):
171
172 def __init__(self, symbols, keyType, valueType):
173 CollectionType.__init__(self, symbols, "map<"+keyType.name+","+valueType.name +">")
174 self.keyType = keyType
175 self.valueType = valueType
176
Marc Slemko27340eb2006-08-10 20:45:55 +0000177 def validate(self):
178 if not isComparableType(self.keyType):
179 raise ErrorException([SymanticsError(self, "key type \""+str(self.keyType)+"\" is not a comparable type.")])
180
Marc Slemkob2039e72006-08-09 01:00:17 +0000181class Set(CollectionType):
182
183 def __init__(self, symbols, valueType):
184 CollectionType.__init__(self, symbols, "set<"+valueType.name+">")
185 self.valueType = valueType
186
Marc Slemko27340eb2006-08-10 20:45:55 +0000187 def validate(self):
188 if not isComparableType(self.valueType):
189 raise ErrorException([SymanticsError(self, "value type \""+str(self.valueType)+"\" is not a comparable type.")])
190
Marc Slemkob2039e72006-08-09 01:00:17 +0000191class List(CollectionType):
192
193 def __init__(self, symbols, valueType):
194 CollectionType.__init__(self, symbols, "list<"+valueType.name+">")
195 self.valueType = valueType
196
197class Enum(Definition):
198
199 def __init__(self, symbols, name, enumDefs):
200 Definition.__init__(self, symbols, name)
201 self.enumDefs = enumDefs
202
203 def validate(self):
204 ids = {}
205 names = {}
206 errors = []
207
208 for enumDef in self.enumDefs:
209
210 if enumDef.name in names:
211 errors.append(SymanticsError(enumDef, self.name+"."+str(enumDef.name)+" already defined at line "+str(names[enumDef.name].start)))
212 else:
213 names[enumDef.name] = enumDef
214
215 if enumDef.id != None:
216 oldEnumDef = ids.get(enumDef.id)
217 if oldEnumDef:
218 errors.append(SymanticsError(enumDef, "enum "+self.name+" \""+str(enumDef.name)+"\" uses constant already assigned to \""+oldEnumDef.name+"\""))
219 else:
220 ids[enumDef.id] = enumDef
221
222 if len(errors):
223 raise ErrorException(errors)
224
225 def assignId(enumDef, currentId, ids):
226 'Finds the next available id number for an enum definition'
227
228 id= currentId + 1
229
230 while id in ids:
231 id += 1
232
233 enumDef.id = id
234
235 ids[enumDef.id] = enumDef
236
237 # assign ids for all enum defs with unspecified ids
238
239 currentId = 0
240
241 for enumDef in self.enumDefs:
242 if not enumDef.id:
243 assignId(enumDef, currentId, ids)
244 currentId = enumDef.id
245
246 def __repr__(self):
247 return str(self)
248
249 def __str__(self):
250 return self.name+"<"+string.join(map(lambda enumDef: str(enumDef), self.enumDefs), ", ")
251
252class EnumDef(Definition):
253
254 def __init__(self, symbols, name, id=None):
255 Definition.__init__(self, symbols, name, id)
256
257 def __repr__(self):
258 return str(self)
259
260 def __str__(self):
261 result = self.name
262 if self.id:
263 result+= ":"+str(self.id)
264 return result
265
266
267class Field(Definition):
268
269 def __init__(self, symbols, type, identifier):
270 Definition.__init__(self, symbols, identifier.name, identifier.id)
271 self.type = type
272 self.identifier = identifier
273
274 def __str__(self):
275 return "<"+str(self.type)+", "+str(self.identifier)+">"
276
277def validateFieldList(fieldList):
278
279 errors = []
280 names = {}
281 ids = {}
282
283 for field in fieldList:
284
285 if field.name in names:
286 oldField = names[field.name]
287 errors.append(SymanticsError(field, "field \""+field.name+"\" already defined at "+str(oldField.start)))
288 else:
289 names[field.name] = field
290
291 if field.id != None:
292 oldField = ids.get(field.id)
293 if oldField:
294 errors.append(SymanticsError(field, "field \""+field.name+"\" uses constant already assigned to \""+oldField.name+"\""))
295 else:
296 ids[field.id] = field
297
298 if len(errors):
299 raise ErrorException(errors)
300
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000301 def assignId(field, currentId, ids):
302 'Finds the next available id number for a field'
303 id= currentId - 1
304
305 while id in ids:
306 id -= 1
307
308 field.id = id
309
310 ids[field.id] = field
311
312 return id
313
314 # assign ids for all fields with unspecified ids
315
316 currentId = 0
317
318 for fields in fieldList:
319 if not field.id:
320 currentId = assignId(field, currentId, ids)
321
Marc Slemkob2039e72006-08-09 01:00:17 +0000322class Struct(Type):
323
324 def __init__(self, symbols, name, fieldList):
325 Type.__init__(self, symbols, name)
326 self.fieldList = fieldList
327
328 def validate(self):
329 validateFieldList(self.fieldList)
330
331 def __str__(self):
332 return self.name+"<"+string.join(map(lambda a: str(a), self.fieldList), ", ")+">"
333
334class Function(Definition):
335
336 def __init__(self, symbols, name, resultType, argFieldList):
337 Definition.__init__(self, symbols, name)
338 self.resultType = resultType
339 self.argFieldList = argFieldList
340
341 def validate(self):
342 validateFieldList(self.argFieldList)
343
344 def __str__(self):
345 return self.name+"("+string.join(map(lambda a: str(a), self.argFieldList), ", ")+") => "+str(self.resultType)
346
347class Service(Definition):
348
349 def __init__(self, symbols, name, functionList):
350 Definition.__init__(self, symbols, name)
351 self.functionList = functionList
352
353 def validate(self):
354
355 errors = []
356 functionNames = {}
357 for function in self.functionList:
358 if function.name in functionNames:
359 oldFunction = functionName[function.name]
360 errors.append(SymanticsError(function, "function "+function.name+" already defined at "+str(oldFunction.start)))
361
362 if len(errors):
363 raise ErrorException(errors)
364
365 def __str__(self):
366 return self.name+"("+string.join(map(lambda a: str(a), self.functionList), ", ")+")"
367
368class Program(object):
369
370 def __init__(self, symbols=None, name="", definitions=None, serviceMap=None, typedefMap=None, enumMap=None, structMap=None, collectionMap=None,
371 primitiveMap=None):
372
373 self.name = name
374
375 if not definitions:
376 definitions = []
377 self.definitions = definitions
378
379 if not serviceMap:
380 serviceMap = {}
381 self.serviceMap = serviceMap
382
383 if not typedefMap:
384 typedefMap = {}
385 self.typedefMap = typedefMap
386
387 if not enumMap:
388 enumMap = {}
389 self.enumMap = enumMap
390
391 if not structMap:
392 structMap = {}
393 self.structMap = structMap
394
395 if not collectionMap:
396 collectionMap = {}
397 self.collectionMap = collectionMap
398
399 if not primitiveMap:
400 primitiveMap = PRIMITIVE_MAP
401 self.primitiveMap = primitiveMap
402
403 def addDefinition(self, definition, definitionMap, definitionTypeName):
404
405 oldDefinition = definitionMap.get(definition.name)
406 if oldDefinition:
407 raise ErrorException([SymanticsError(definition, definitionTypeName+" "+definition.name+" is already defined at "+str(oldDefinition.start))])
408 else:
409 definitionMap[definition.name] = definition
410
411 # keep an ordered list of definitions so that stub/skel generators can determine the original order
412
413 self.definitions.append(definition)
414
415 def addStruct(self, struct):
416 self.addDefinition(struct, self.structMap, "struct")
417
418 def addTypedef(self, typedef):
419 self.addDefinition(typedef, self.typedefMap, "typedef")
420
421 def addEnum(self, enum):
422 self.addDefinition(enum, self.enumMap, "enum")
423
424 def addService(self, service):
425 self.addDefinition(service, self.serviceMap, "service")
426
427 def addCollection(self, collection):
428 if collection.name in self.collectionMap:
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000429 return self.collectionMap[collection.name]
Marc Slemkob2039e72006-08-09 01:00:17 +0000430 else:
431 self.collectionMap[collection.name] = collection
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000432 return collection
Marc Slemkob2039e72006-08-09 01:00:17 +0000433
434 def getType(self, parent, symbol):
435 """ Get the type definition for a symbol"""
436
437 typeName = None
438
439 if isinstance(symbol, Type):
440 return symbol
441 elif isinstance(symbol, Field):
442 typeName = symbol.type.name
443 elif isinstance(symbol, Identifier):
444 typeName = symbol.name
445 else:
446 raise ErrorException([SymanticsError(parent, "unknown symbol \""+str(symbol)+"\"")])
447
448 for map in (self.primitiveMap, self.collectionMap, self.typedefMap, self.enumMap, self.structMap):
449 if typeName in map:
450 return map[typeName]
451
452 raise ErrorException([SymanticsError(parent, "\""+typeName+"\" is not defined.")])
453
454 def hasType(self, parent, symbol):
455 """ Determine if a type definition exists for the symbol"""
456
457 return self.getType(parent, symbol) == True
458
459 def validate(self):
460
461 errors = []
462
463 # Verify that struct fields types, collection key and element types, and typedef defined types exists and replaces
464 # type names with references to the type objects
465
466 for struct in self.structMap.values():
467 for field in struct.fieldList:
468 try:
469 field.type = self.getType(struct, field)
470 except ErrorException, e:
471 errors+= e.errors
472
473 for collection in self.collectionMap.values():
474 try:
475 if isinstance(collection, Map):
476 collection.keyType = self.getType(collection, collection.keyType)
477
478 collection.valueType = self.getType(collection, collection.valueType)
Marc Slemko27340eb2006-08-10 20:45:55 +0000479
480 collection.validate()
Marc Slemkob2039e72006-08-09 01:00:17 +0000481
482 except ErrorException, e:
483 errors+= e.errors
484
485 for typedef in self.typedefMap.values():
486 try:
487 typedef.definitionType = self.getType(self, typedef.definitionType)
488
489 except ErrorException, e:
490 errors+= e.errors
491
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000492 # Verify that service function result and arg list types exist and replace type name with reference to definition
Marc Slemkob2039e72006-08-09 01:00:17 +0000493
494 for service in self.serviceMap.values():
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000495
Marc Slemkob2039e72006-08-09 01:00:17 +0000496 for function in service.functionList:
497 try:
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000498 function.resultType = self.getType(service, function.resultType)
Marc Slemkob2039e72006-08-09 01:00:17 +0000499 except ErrorException, e:
500 errors+= e.errors
501
502 for field in function.argFieldList:
503 try:
504 field.type = self.getType(function, field)
505 except ErrorException, e:
506 errors+= e.errors
507
508 if len(errors):
509 raise ErrorException(errors)
Marc Slemkob2039e72006-08-09 01:00:17 +0000510
511class Parser(object):
512
513 reserved = ("BYTE",
514 # "CONST",
515 "DOUBLE",
516 "ENUM",
517 # "EXCEPTION",
518 # "EXTENDS",
519 "I08",
520 "I16",
521 "I32",
522 "I64",
523 "LIST",
524 "MAP",
525 "SERVICE",
526 "SET",
527 # "STATIC",
528 "STRING",
529 "STRUCT",
530 # "SYNCHRONIZED",
531 "TYPEDEF",
532 "U08",
533 "U16",
534 "U32",
535 "U64",
536 "UTF16",
537 "UTF8",
538 "VOID"
539 )
540
541 tokens = reserved + (
542 # Literals (identifier, integer constant, float constant, string constant, char const)
543 'ID', 'ICONST', 'SCONST', 'FCONST',
544 # Operators default=, optional*, variable...
545 'ASSIGN', #'OPTIONAL', 'ELLIPSIS',
546 # Delimeters ( ) { } < > , . ; :
547 'LPAREN', 'RPAREN',
548 'LBRACE', 'RBRACE',
549 'LANGLE', 'RANGLE',
550 'COMMA' #, 'PERIOD', 'SEMI' , 'COLON'
551 )
552
553 precendence = ()
554
555 reserved_map = {}
556
557 for r in reserved:
558 reserved_map[r.lower()] = r
559
560 def t_ID(self, t):
561 r'[A-Za-z_][\w_]*'
562 t.type = self.reserved_map.get(t.value,"ID")
563 return t
564
565 # Completely ignored characters
566 t_ignore = ' \t\x0c'
567
568# t_OPTIONAL = r'\*'
569 t_ASSIGN = r'='
570
571 # Delimeters
572 t_LPAREN = r'\('
573 t_RPAREN = r'\)'
574 t_LANGLE = r'\<'
575 t_RANGLE = r'\>'
576 t_LBRACE = r'\{'
577 t_RBRACE = r'\}'
578 t_COMMA = r','
579# t_PERIOD = r'\.'
580# t_SEMI = r';'
581# t_COLON = r':'
582# t_ELLIPSIS = r'\.\.\.'
583
584 # Integer literal
585 t_ICONST = r'\d+([uU]|[lL]|[uU][lL]|[lL][uU])?'
586
587 # Floating literal
588 t_FCONST = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?'
589
590 # String literal
591 t_SCONST = r'\"([^\\\n]|(\\.))*?\"'
592
593 # Comments
594 def t_comment(self, t):
595 r'(?:/\*(.|\n)*?\*/)|(?://[^\n]*\n)'
596 t.lineno += t.value.count('\n')
597
598 def t_error(self, t):
599 print "Illegal character %s" % repr(t.value[0])
600 t.skip(1)
601
602 # Newlines
603 def t_newline(self, t):
604 r'\n+'
605 t.lineno += t.value.count("\n")
606
607 def p_program(self, p):
608 'program : definitionlist'
609 pass
610
611 def p_definitionlist_1(self, p):
612 'definitionlist : definitionlist definition'
613 pass
614
615 def p_definitionlist_2(self, p):
616 'definitionlist :'
617 pass
618
619 def p_definition_1(self, p):
620 'definition : typedef'
621 self.pdebug("p_definition_1", p)
622 p[0] = p[1]
623 try:
624 self.program.addTypedef(p[0])
625 except ErrorException, e:
626 self.errors+= e.errors
627
628 def p_definition_2(self, p):
629 'definition : enum'
630 self.pdebug("p_definition_2", p)
631 p[0] = p[1]
632 try:
633 self.program.addEnum(p[0])
634 except ErrorException, e:
635 self.errors+= e.errors
636
637 def p_definition_3(self, p):
638 'definition : struct'
639 self.pdebug("p_definition_3", p)
640 p[0] = p[1]
641 try:
642 self.program.addStruct(p[0])
643 except ErrorException, e:
644 self.errors+= e.errors
645
646 def p_definition_4(self, p):
647 'definition : service'
648 self.pdebug("p_definition_4", p)
649 p[0] = p[1]
650 try:
651 self.program.addService(p[0])
652 except ErrorException, e:
653 self.errors+= e.errors
654
655 def p_typedef(self, p):
656 'typedef : TYPEDEF definitiontype ID'
657 self.pdebug("p_typedef", p)
658 p[0] = TypeDef(p, p[3], p[2])
659 try:
660 p[0].validate()
661
662 except ErrorException, e:
663 self.errors+= e.errors
664
665# def p_definition_or_referencye_type_1(self, p):
666# XXX need to all typedef struct foo foo_t by allowing references
667# pass
668
669 def p_enum(self, p):
670 'enum : ENUM ID LBRACE enumdeflist RBRACE'
671 self.pdebug("p_enum", p)
672 p[0] = Enum(p, p[2], p[4])
673
674 try:
675 p[0].validate()
676 except ErrorException, e:
677 self.errors+= e.errors
678
679 def p_enumdeflist_1(self, p):
680 'enumdeflist : enumdeflist COMMA enumdef'
681 self.pdebug("p_enumdeflist_1", p)
682 p[0] = p[1] + (p[3],)
683
684 def p_enumdeflist_2(self, p):
685 'enumdeflist : enumdef'
686 self.pdebug("p_enumdeflist_2", p)
687 p[0] = (p[1],)
688
689 def p_enumdef_0(self, p):
690 'enumdef : ID ASSIGN ICONST'
691 self.pdebug("p_enumdef_0", p)
692 p[0] = EnumDef(p, p[1], int(p[3]))
693
694 def p_enumdef_1(self, p):
695 'enumdef : ID'
696 self.pdebug("p_enumdef_1", p)
697 p[0] = EnumDef(p, p[1])
698
699 def p_struct(self, p):
700 'struct : STRUCT ID LBRACE fieldlist RBRACE'
701 self.pdebug("p_struct", p)
702 p[0] = Struct(p, p[2], p[4])
703
704 try:
705 p[0].validate()
706 except ErrorException, e:
707 self.errors+= e.errors
708
709 def p_service(self, p):
710 'service : SERVICE ID LBRACE functionlist RBRACE'
711 self.pdebug("p_service", p)
712 p[0] = Service(p, p[2], p[4])
713 try:
714 p[0].validate()
715 except ErrorException, e:
716 self.errors+= e.errors
717
718 def p_functionlist_1(self, p):
719 'functionlist : functionlist function'
720 self.pdebug("p_functionlist_1", p)
721 p[0] = p[1] + (p[2],)
722
723 def p_functionlist_2(self, p):
724 'functionlist :'
725 self.pdebug("p_functionlist_2", p)
726 p[0] = ()
727
728 def p_function(self, p):
729 'function : functiontype functionmodifiers ID LPAREN fieldlist RPAREN'
730 self.pdebug("p_function", p)
731 p[0] = Function(p, p[3], p[1], p[5])
732 try:
733 p[0].validate()
734 except ErrorException, e:
735 self.errors+= e.errors
736
737 def p_functionmodifiers(self, p):
738 'functionmodifiers :'
739 self.pdebug("p_functionmodifiers", p)
740 p[0] = ()
741
742 def p_fieldlist_1(self, p):
743 'fieldlist : fieldlist COMMA field'
744 self.pdebug("p_fieldlist_1", p)
745 p[0] = p[1] + (p[3],)
746
747 def p_fieldlist_2(self, p):
748 'fieldlist : field'
749 self.pdebug("p_fieldlist_2", p)
750 p[0] = (p[1],)
751
752 def p_fieldlist_3(self, p):
753 'fieldlist :'
754 self.pdebug("p_fieldlist_3", p)
755 p[0] = ()
756
757 def p_field_1(self, p):
758 'field : fieldtype ID ASSIGN ICONST'
759 self.pdebug("p_field_1", p)
760 p[0] = Field(p, p[1], Identifier(None, p[2], int(p[4])))
761
762 def p_field_2(self, p):
763 'field : fieldtype ID'
764 self.pdebug("p_field_2", p)
765 p[0] = Field(p, p[1], Identifier(None, p[2]))
766
767 def p_definitiontype_1(self, p):
768 'definitiontype : basetype'
769 self.pdebug("p_definitiontype_1", p)
770 p[0] = p[1]
771
772 def p_definitiontype_2(self, p):
773 'definitiontype : collectiontype'
774 self.pdebug("p_definitiontype_2", p)
775 p[0] = p[1]
776
777 def p_functiontype_1(self, p):
778 'functiontype : fieldtype'
779 self.pdebug("p_functiontype_1", p)
780 p[0] = p[1]
781
782 def p_functiontype_2(self, p):
783 'functiontype : VOID'
784 self.pdebug("p_functiontype_2", p)
785 p[0] = self.program.primitiveMap[p[1].lower()]
786
787 def p_fieldtype_1(self, p):
788 'fieldtype : ID'
789 self.pdebug("p_fieldtype_1", p)
790 p[0] = Identifier(p, p[1])
791
792 def p_fieldtype_2(self, p):
793 'fieldtype : basetype'
794 self.pdebug("p_fieldtype_2", p)
795 p[0] = p[1]
796
797 def p_fieldtype_3(self, p):
798 'fieldtype : collectiontype'
799 self.pdebug("p_fieldtype_3", p)
800 p[0] = p[1]
801
802 def p_basetype_1(self, p):
803 'basetype : STRING'
804 self.pdebug("p_basetype_1", p)
805 p[0] = self.program.primitiveMap[p[1].lower()]
806
807 def p_basetype_2(self, p):
808 'basetype : BYTE'
809 self.pdebug("p_basetype_2", p)
810 p[0] = self.program.primitiveMap[p[1].lower()]
811
812 def p_basetype_3(self, p):
813 'basetype : I08'
814 self.pdebug("p_basetype_3", p)
815 p[0] = self.program.primitiveMap[p[1].lower()]
816
817 def p_basetype_4(self, p):
818 'basetype : U08'
819 self.pdebug("p_basetype_4", p)
820 p[0] = self.program.primitiveMap[p[1].lower()]
821
822 def p_basetype_5(self, p):
823 'basetype : I16'
824 self.pdebug("p_basetype_5", p)
825 p[0] = self.program.primitiveMap[p[1].lower()]
826
827 def p_basetype_6(self, p):
828 'basetype : U16'
829 self.pdebug("p_basetype_6", p)
830 p[0] = self.program.primitiveMap[p[1].lower()]
831
832 def p_basetype_7(self, p):
833 'basetype : I32'
834 self.pdebug("p_basetype_7", p)
835 p[0] = self.program.primitiveMap[p[1].lower()]
836
837 def p_basetype_8(self, p):
838 'basetype : U32'
839 self.pdebug("p_basetype_8", p)
840 p[0] = self.program.primitiveMap[p[1].lower()]
841
842 def p_basetype_9(self, p):
843 'basetype : I64'
844 self.pdebug("p_basetype_9", p)
845 p[0] = self.program.primitiveMap[p[1].lower()]
846
847 def p_basetype_10(self, p):
848 'basetype : U64'
849 self.pdebug("p_basetype_10", p)
850 p[0] = self.program.primitiveMap[p[1].lower()]
851
852 def p_basetype_11(self, p):
853 'basetype : UTF8'
854 self.pdebug("p_basetype_11", p)
855 p[0] = self.program.primitiveMap[p[1].lower()]
856
857 def p_basetype_12(self, p):
858 'basetype : UTF16'
859 self.pdebug("p_basetype_12", p)
860 p[0] = self.program.primitiveMap[p[1].lower()]
861
862 def p_basetype_13(self, p):
863 'basetype : DOUBLE'
864 self.pdebug("p_basetype_13", p)
865 p[0] = self.program.primitiveMap[p[1].lower()]
866
867 def p_collectiontype_1(self, p):
868 'collectiontype : maptype'
869 self.pdebug("p_collectiontype_1", p)
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000870 p[0] = self.program.addCollection(p[1])
Marc Slemkob2039e72006-08-09 01:00:17 +0000871
872 def p_collectiontype_2(self, p):
873 'collectiontype : settype'
874 self.pdebug("p_collectiontype_2", p)
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000875 p[0] = self.program.addCollection(p[1])
Marc Slemkob2039e72006-08-09 01:00:17 +0000876
877 def p_collectiontype_3(self, p):
878 'collectiontype : listtype'
879 self.pdebug("p_collectiontype_3", p)
Marc Slemkoc0e07a22006-08-09 23:34:57 +0000880 p[0] = self.program.addCollection(p[1])
Marc Slemkob2039e72006-08-09 01:00:17 +0000881
882 def p_maptype(self, p):
883 'maptype : MAP LANGLE fieldtype COMMA fieldtype RANGLE'
884 self.pdebug("p_maptype", p)
885 p[0] = Map(p, p[3], p[5])
886
887 def p_settype(self, p):
888 'settype : SET LANGLE fieldtype RANGLE'
889 self.pdebug("p_settype", p)
890 p[0] = Set(p, p[3])
891
892 def p_listtype(self, p):
893 'listtype : LIST LANGLE fieldtype RANGLE'
894 self.pdebug("p_listtype", p)
Marc Slemkoc4eb9e82006-08-10 03:29:29 +0000895 p[0] = List(p, p[3])
Marc Slemkob2039e72006-08-09 01:00:17 +0000896
897 def p_error(self, p):
Marc Slemko27340eb2006-08-10 20:45:55 +0000898 # p_error is called with an empty token if eof was encountered unexpectedly.
899 if not p:
900 self.errors.append(SyntaxError("Unexpected end of file"))
901 else:
902 self.errors.append(SyntaxError(p))
Marc Slemkob2039e72006-08-09 01:00:17 +0000903
904 def pdebug(self, name, p):
905 if self.debug:
906 print(name+"("+string.join(map(lambda t: "<<"+str(t)+">>", p), ", ")+")")
907
908 def __init__(self, **kw):
909 self.debug = kw.get('debug', 0)
910 self.names = { }
911 self.program = Program()
912 self.errors = []
913
914 try:
915 modname = os.path.split(os.path.splitext(__file__)[0])[1] + "_" + self.__class__.__name__
916 except:
917 modname = "parser"+"_"+self.__class__.__name__
918 self.debugfile = modname + ".dbg"
919 self.tabmodule = modname + "_" + "parsetab"
920 #print self.debugfile, self.tabmodule
921
922 # Build the lexer and parser
923 lex.lex(module=self, debug=self.debug)
924 yacc.yacc(module=self,
925 debug=self.debug,
926 debugfile=self.debugfile,
927 tabmodule=self.tabmodule)
928
929 def parsestring(self, s, filename=""):
930 yacc.parse(s)
931
932 if len(self.errors) == 0:
933 try:
934 self.program.validate()
935 except ErrorException, e:
936 self.errors+= e.errors
937
938 if len(self.errors):
939 for error in self.errors:
940 print(filename+":"+str(error))
941
942 def parse(self, filename, doPickle=True):
943
944 f = file(filename, "r")
945
946 self.parsestring(f.read(), filename)
947
948 if len(self.errors) == 0 and doPickle:
949
950 outf = file(os.path.splitext(filename)[0]+".thyc", "w")
951
952 pickle.dump(self.program, outf)