1 import Martel
2 from Martel import IterParser, RecordReader, LAX, Parser
3
4
6 return IterParser.IterHeaderFooter(
7 Martel.Re(r"a*\R").make_parser(),
8 RecordReader.CountLines,
9 (1,),
10
11 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(debug_level = 1),
12 RecordReader.CountLines,
13 (2,),
14
15 Martel.Re(r"d*\R").make_parser(),
16 RecordReader.CountLines,
17 (1,),
18
19 "spam")
20
22 ip = gen_iterator()
23
24 lines = ["aaaaaaaaa",
25 "b",
26 "c",
27 "bb",
28 "cc",
29 "bbb",
30 "ccc",
31 "d"]
32 text = "\n".join(lines) + "\n"
33
34
35 i = 1
36 for x in ip.iterateString(text):
37 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n"
38 i = i + 1
39
41 ip = IterParser.IterHeaderFooter(
42 None,
43 None,
44 None,
45
46 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(),
47 RecordReader.CountLines,
48 (2,),
49
50 Martel.Re(r"d*\R").make_parser(),
51 RecordReader.CountLines,
52 (1,),
53
54 "spam")
55
56 lines = ["b",
57 "c",
58 "bb",
59 "cc",
60 "bbb",
61 "ccc",
62 "d"]
63 text = "\n".join(lines) + "\n"
64
65 i = 1
66 for x in ip.iterateString(text):
67 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n"
68 i = i + 1
69
71 ip = IterParser.IterHeaderFooter(
72 None,
73 None,
74 None,
75
76 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(),
77 RecordReader.CountLines,
78 (2,),
79
80 None,
81 None,
82 None,
83
84 "spam")
85
86 lines = ["b",
87 "c",
88 "bb",
89 "cc",
90 "bbb",
91 "ccc",
92 ]
93 text = "\n".join(lines) + "\n"
94
95 i = 1
96 for x in ip.iterateString(text):
97 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n"
98 i = i + 1
99
101 ip = IterParser.IterHeaderFooter(
102 Martel.Re(r"a*\R").make_parser(),
103 RecordReader.CountLines,
104 (1,),
105
106 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(),
107 RecordReader.CountLines,
108 (2,),
109
110 None,
111 None,
112 None,
113
114 "spam")
115
116 lines = ["aaaaaaaaa",
117 "b",
118 "c",
119 "bb",
120 "cc",
121 "bbb",
122 "ccc",
123 ]
124 text = "\n".join(lines) + "\n"
125
126 i = 1
127 for x in ip.iterateString(text):
128 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n"
129 i = i + 1
130
131
152
173
194
215
216
217
219 ip = IterParser.IterRecords(
220 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(),
221 RecordReader.CountLines,
222 (2,),
223
224 "spam")
225
226 lines = ["b",
227 "c",
228 "bb",
229 "cc",
230 "bbb",
231 "ccc",
232 ]
233 text = "\n".join(lines) + "\n"
234 i = 1
235 for x in ip.iterateString(text):
236 assert x["spam"][0] == "b" * i + "\n" + "c" * i + "\n"
237 i = i + 1
238
239
241
242 ip = IterParser.IterRecords(
243 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(debug_level = 1),
244 RecordReader.CountLines,
245 (2,),
246
247 "spam")
248
249 lines = ["b",
250 "-",
251 "bb",
252 "cc",
253 "bbb",
254 "ccc",
255 ]
256 text = "\n".join(lines) + "\n"
257 try:
258 for x in ip.iterateString(text):
259 pass
260 except Parser.ParserPositionException, exc:
261 assert exc.pos == 2, exc.pos
262
264
265 ip = IterParser.IterRecords(
266 Martel.Group("spam", Martel.Re(r"b*\Rc*\R")).make_parser(debug_level = 1),
267 RecordReader.CountLines,
268 (2,),
269
270 "spam")
271
272 lines = ["b",
273 "c",
274 "b-",
275 "cc",
276 "bbb",
277 "ccc",
278 ]
279 text = "\n".join(lines) + "\n"
280 try:
281 for x in ip.iterateString(text):
282 pass
283 except Parser.ParserPositionException, exc:
284 assert exc.pos == 5, exc.pos
285
287 exp = Martel.ParseRecords("dataset", {},
288 Martel.Group("spam", Martel.Re(r"a*\R")),
289 RecordReader.CountLines, (1,))
290 iterator = exp.make_iterator("spam")
291 assert isinstance(iterator, IterParser.IterRecords)
292 lines = []
293 for i in range(0, 10):
294 lines.append("a" * i + "\n")
295 text = "".join(lines)
296
297 i = 0
298 for rec in iterator.iterateString(text):
299 assert len(rec["spam"][0][:-1]) == i, (i, rec["spam"][0])
300 i = i + 1
301 assert i == 10
302
303
305 exp = Martel.HeaderFooter("dataset", {},
306 Martel.Group("header", Martel.Re(r"(a*\R)*")),
307 RecordReader.Until, ("b",),
308
309 Martel.Group("record", Martel.Re(r"(b*\R)*")),
310 RecordReader.Until, ("c",),
311
312 Martel.Group("footer", Martel.Re(r"(c*\R)*")),
313 RecordReader.Everything, (),)
314
315 iterator = exp.make_iterator("record")
316 assert isinstance(iterator, IterParser.IterHeaderFooter), iterator
317 lines = ["a"
318 "aa",
319 "aaaaaaa",
320 "b",
321 "bb",
322 "bbbb",
323 "bbbbbbbb",
324 "bbbbbbbbbbbbbbbb",
325 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
326 "cccc",
327 "cc",
328 "c",
329 ]
330
331 text = "\n".join(lines) + "\n"
332
333 i = 0
334 for rec in iterator.iterateString(text):
335 i = i + 1
336 assert i == 1, i
337
339 exp = Martel.ParseRecords("dataset", {},
340 Martel.Group("record", Martel.Re(r"(b*\R)*a\R")),
341 RecordReader.EndsWith, ("a",))
342 lines = [
343 "bbb",
344 "bb",
345 "a",
346 "b",
347 "a",
348 "a",
349 ]
350 text = "\n".join(lines) + "\n"
351
352 iterator = exp.make_iterator("record")
353
354 for x in iterator.iterateString(text):
355 pass
356
357
358 lines.append("c")
359 text = "\n".join(lines) + "\n"
360 try:
361 for x in iterator.iterateString(text):
362 pass
363 raise AssertionError
364 except Parser.ParserPositionException, exc:
365 assert exc.pos == 15, exc.pos
366
368
369 exp = Martel.HeaderFooter("dataset", {},
370 None, None, None,
371 Martel.Group("record", Martel.Re(r"(b*\R)*a\R")),
372 RecordReader.EndsWith, ("a",),
373 None, None, None
374 )
375 lines = [
376 "bbb",
377 "bb",
378 "a",
379 "b",
380 "a",
381 "a",
382 ]
383 text = "\n".join(lines) + "\n"
384
385 iterator = exp.make_iterator("record")
386
387 for x in iterator.iterateString(text):
388 pass
389
390
391 lines.append("c")
392 text = "\n".join(lines) + "\n"
393 try:
394 for x in iterator.iterateString(text):
395 pass
396 raise AssertionError
397 except Parser.ParserPositionException, exc:
398 assert exc.pos == 15, exc.pos
399
401
402 exp = Martel.HeaderFooter("dataset", {},
403 None, None, None,
404
405 Martel.Group("record", Martel.Re(r"(b*\R)*a\R")),
406 RecordReader.EndsWith, ("a",),
407
408 Martel.Group("footer", Martel.Re(r"c\R")),
409 RecordReader.CountLines, (1,)
410 )
411 lines = [
412 "bbb",
413 "bb",
414 "a",
415 "b",
416 "a",
417 "a",
418 "c",
419 ]
420 text = "\n".join(lines) + "\n"
421
422 iterator = exp.make_iterator("record")
423
424 for x in iterator.iterateString(text):
425 pass
426
427
428 lines.pop()
429 text = "\n".join(lines) + "\n"
430 try:
431 for x in iterator.iterateString(text):
432 pass
433 raise AssertionError
434 except Parser.ParserPositionException, exc:
435 assert exc.pos == 15, exc.pos
436
437
458
459 if __name__ == "__main__":
460 test()
461