-
-
Notifications
You must be signed in to change notification settings - Fork 31.5k
/
Copy pathcodegen.c
6388 lines (5748 loc) · 206 KB
/
codegen.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* This file implements the compiler's code generation stage, which
* produces a sequence of pseudo-instructions from an AST.
*
* The primary entry point is _PyCodegen_Module() for modules, and
* _PyCodegen_Expression() for expressions.
*
* CAUTION: The VISIT_* macros abort the current function when they
* encounter a problem. So don't invoke them when there is memory
* which needs to be released. Code blocks are OK, as the compiler
* structure takes care of releasing those. Use the arena to manage
* objects.
*/
#include "Python.h"
#include "opcode.h"
#include "pycore_ast.h" // _PyAST_GetDocString()
#define NEED_OPCODE_TABLES
#include "pycore_opcode_utils.h"
#undef NEED_OPCODE_TABLES
#include "pycore_c_array.h" // _Py_c_array_t
#include "pycore_code.h" // COMPARISON_LESS_THAN
#include "pycore_compile.h"
#include "pycore_instruction_sequence.h" // _PyInstructionSequence_NewLabel()
#include "pycore_intrinsics.h"
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_object.h" // _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS
#include "pycore_pystate.h" // _Py_GetConfig()
#include "pycore_symtable.h" // PySTEntryObject
#include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString
#define NEED_OPCODE_METADATA
#include "pycore_opcode_metadata.h" // _PyOpcode_opcode_metadata, _PyOpcode_num_popped/pushed
#undef NEED_OPCODE_METADATA
#include <stdbool.h>
#define COMP_GENEXP 0
#define COMP_LISTCOMP 1
#define COMP_SETCOMP 2
#define COMP_DICTCOMP 3
#undef SUCCESS
#undef ERROR
#define SUCCESS 0
#define ERROR -1
#define RETURN_IF_ERROR(X) \
do { \
if ((X) == -1) { \
return ERROR; \
} \
} while (0)
#define RETURN_IF_ERROR_IN_SCOPE(C, CALL) \
do { \
if ((CALL) < 0) { \
_PyCompile_ExitScope((C)); \
return ERROR; \
} \
} while (0)
struct _PyCompiler;
typedef struct _PyCompiler compiler;
#define INSTR_SEQUENCE(C) _PyCompile_InstrSequence(C)
#define FUTURE_FEATURES(C) _PyCompile_FutureFeatures(C)
#define SYMTABLE(C) _PyCompile_Symtable(C)
#define SYMTABLE_ENTRY(C) _PyCompile_SymtableEntry(C)
#define OPTIMIZATION_LEVEL(C) _PyCompile_OptimizationLevel(C)
#define IS_INTERACTIVE_TOP_LEVEL(C) _PyCompile_IsInteractiveTopLevel(C)
#define SCOPE_TYPE(C) _PyCompile_ScopeType(C)
#define QUALNAME(C) _PyCompile_Qualname(C)
#define METADATA(C) _PyCompile_Metadata(C)
typedef _PyInstruction instruction;
typedef _PyInstructionSequence instr_sequence;
typedef _Py_SourceLocation location;
typedef _PyJumpTargetLabel jump_target_label;
typedef _PyCompile_FBlockInfo fblockinfo;
#define LOCATION(LNO, END_LNO, COL, END_COL) \
((const _Py_SourceLocation){(LNO), (END_LNO), (COL), (END_COL)})
#define LOC(x) SRC_LOCATION_FROM_AST(x)
#define NEW_JUMP_TARGET_LABEL(C, NAME) \
jump_target_label NAME = _PyInstructionSequence_NewLabel(INSTR_SEQUENCE(C)); \
if (!IS_JUMP_TARGET_LABEL(NAME)) { \
return ERROR; \
}
#define USE_LABEL(C, LBL) \
RETURN_IF_ERROR(_PyInstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id))
static const int compare_masks[] = {
[Py_LT] = COMPARISON_LESS_THAN,
[Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS,
[Py_EQ] = COMPARISON_EQUALS,
[Py_NE] = COMPARISON_NOT_EQUALS,
[Py_GT] = COMPARISON_GREATER_THAN,
[Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
};
int
_Py_CArray_Init(_Py_c_array_t* array, int item_size, int initial_num_entries) {
memset(array, 0, sizeof(_Py_c_array_t));
array->item_size = item_size;
array->initial_num_entries = initial_num_entries;
return 0;
}
void
_Py_CArray_Fini(_Py_c_array_t* array)
{
if (array->array) {
PyMem_Free(array->array);
array->allocated_entries = 0;
}
}
int
_Py_CArray_EnsureCapacity(_Py_c_array_t *c_array, int idx)
{
void *arr = c_array->array;
int alloc = c_array->allocated_entries;
if (arr == NULL) {
int new_alloc = c_array->initial_num_entries;
if (idx >= new_alloc) {
new_alloc = idx + c_array->initial_num_entries;
}
arr = PyMem_Calloc(new_alloc, c_array->item_size);
if (arr == NULL) {
PyErr_NoMemory();
return ERROR;
}
alloc = new_alloc;
}
else if (idx >= alloc) {
size_t oldsize = alloc * c_array->item_size;
int new_alloc = alloc << 1;
if (idx >= new_alloc) {
new_alloc = idx + c_array->initial_num_entries;
}
size_t newsize = new_alloc * c_array->item_size;
if (oldsize > (SIZE_MAX >> 1)) {
PyErr_NoMemory();
return ERROR;
}
assert(newsize > 0);
void *tmp = PyMem_Realloc(arr, newsize);
if (tmp == NULL) {
PyErr_NoMemory();
return ERROR;
}
alloc = new_alloc;
arr = tmp;
memset((char *)arr + oldsize, 0, newsize - oldsize);
}
c_array->array = arr;
c_array->allocated_entries = alloc;
return SUCCESS;
}
typedef struct {
// A list of strings corresponding to name captures. It is used to track:
// - Repeated name assignments in the same pattern.
// - Different name assignments in alternatives.
// - The order of name assignments in alternatives.
PyObject *stores;
// If 0, any name captures against our subject will raise.
int allow_irrefutable;
// An array of blocks to jump to on failure. Jumping to fail_pop[i] will pop
// i items off of the stack. The end result looks like this (with each block
// falling through to the next):
// fail_pop[4]: POP_TOP
// fail_pop[3]: POP_TOP
// fail_pop[2]: POP_TOP
// fail_pop[1]: POP_TOP
// fail_pop[0]: NOP
jump_target_label *fail_pop;
// The current length of fail_pop.
Py_ssize_t fail_pop_size;
// The number of items on top of the stack that need to *stay* on top of the
// stack. Variable captures go beneath these. All of them will be popped on
// failure.
Py_ssize_t on_top;
} pattern_context;
static int codegen_nameop(compiler *, location, identifier, expr_context_ty);
static int codegen_visit_stmt(compiler *, stmt_ty);
static int codegen_visit_keyword(compiler *, keyword_ty);
static int codegen_visit_expr(compiler *, expr_ty);
static int codegen_augassign(compiler *, stmt_ty);
static int codegen_annassign(compiler *, stmt_ty);
static int codegen_subscript(compiler *, expr_ty);
static int codegen_slice_two_parts(compiler *, expr_ty);
static int codegen_slice(compiler *, expr_ty);
static int codegen_body(compiler *, location, asdl_stmt_seq *, bool);
static int codegen_with(compiler *, stmt_ty);
static int codegen_async_with(compiler *, stmt_ty);
static int codegen_with_inner(compiler *, stmt_ty, int);
static int codegen_async_with_inner(compiler *, stmt_ty, int);
static int codegen_async_for(compiler *, stmt_ty);
static int codegen_call_simple_kw_helper(compiler *c,
location loc,
asdl_keyword_seq *keywords,
Py_ssize_t nkwelts);
static int codegen_call_helper_impl(compiler *c, location loc,
int n, /* Args already pushed */
asdl_expr_seq *args,
PyObject *injected_arg,
asdl_keyword_seq *keywords);
static int codegen_call_helper(compiler *c, location loc,
int n, asdl_expr_seq *args,
asdl_keyword_seq *keywords);
static int codegen_try_except(compiler *, stmt_ty);
static int codegen_try_star_except(compiler *, stmt_ty);
static int codegen_sync_comprehension_generator(
compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type,
int iter_on_stack);
static int codegen_async_comprehension_generator(
compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type,
int iter_on_stack);
static int codegen_pattern(compiler *, pattern_ty, pattern_context *);
static int codegen_match(compiler *, stmt_ty);
static int codegen_pattern_subpattern(compiler *,
pattern_ty, pattern_context *);
static int codegen_make_closure(compiler *c, location loc,
PyCodeObject *co, Py_ssize_t flags);
/* Add an opcode with an integer argument */
static int
codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc)
{
/* oparg value is unsigned, but a signed C int is usually used to store
it in the C code (like Python/ceval.c).
Limit to 32-bit signed C int (rather than INT_MAX) for portability.
The argument of a concrete bytecode instruction is limited to 8-bit.
EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */
int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int);
assert(!IS_ASSEMBLER_OPCODE(opcode));
return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc);
}
#define ADDOP_I(C, LOC, OP, O) \
RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC)))
#define ADDOP_I_IN_SCOPE(C, LOC, OP, O) \
RETURN_IF_ERROR_IN_SCOPE(C, codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC)))
static int
codegen_addop_noarg(instr_sequence *seq, int opcode, location loc)
{
assert(!OPCODE_HAS_ARG(opcode));
assert(!IS_ASSEMBLER_OPCODE(opcode));
return _PyInstructionSequence_Addop(seq, opcode, 0, loc);
}
#define ADDOP(C, LOC, OP) \
RETURN_IF_ERROR(codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC)))
#define ADDOP_IN_SCOPE(C, LOC, OP) \
RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC)))
static int
codegen_addop_load_const(compiler *c, location loc, PyObject *o)
{
Py_ssize_t arg = _PyCompile_AddConst(c, o);
if (arg < 0) {
return ERROR;
}
ADDOP_I(c, loc, LOAD_CONST, arg);
return SUCCESS;
}
#define ADDOP_LOAD_CONST(C, LOC, O) \
RETURN_IF_ERROR(codegen_addop_load_const((C), (LOC), (O)))
#define ADDOP_LOAD_CONST_IN_SCOPE(C, LOC, O) \
RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_load_const((C), (LOC), (O)))
/* Same as ADDOP_LOAD_CONST, but steals a reference. */
#define ADDOP_LOAD_CONST_NEW(C, LOC, O) \
do { \
PyObject *__new_const = (O); \
if (__new_const == NULL) { \
return ERROR; \
} \
if (codegen_addop_load_const((C), (LOC), __new_const) < 0) { \
Py_DECREF(__new_const); \
return ERROR; \
} \
Py_DECREF(__new_const); \
} while (0)
static int
codegen_addop_o(compiler *c, location loc,
int opcode, PyObject *dict, PyObject *o)
{
Py_ssize_t arg = _PyCompile_DictAddObj(dict, o);
RETURN_IF_ERROR(arg);
ADDOP_I(c, loc, opcode, arg);
return SUCCESS;
}
#define ADDOP_N(C, LOC, OP, O, TYPE) \
do { \
assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \
int ret = codegen_addop_o((C), (LOC), (OP), \
METADATA(C)->u_ ## TYPE, (O)); \
Py_DECREF((O)); \
RETURN_IF_ERROR(ret); \
} while (0)
#define ADDOP_N_IN_SCOPE(C, LOC, OP, O, TYPE) \
do { \
assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */ \
int ret = codegen_addop_o((C), (LOC), (OP), \
METADATA(C)->u_ ## TYPE, (O)); \
Py_DECREF((O)); \
RETURN_IF_ERROR_IN_SCOPE((C), ret); \
} while (0)
#define LOAD_METHOD -1
#define LOAD_SUPER_METHOD -2
#define LOAD_ZERO_SUPER_ATTR -3
#define LOAD_ZERO_SUPER_METHOD -4
static int
codegen_addop_name(compiler *c, location loc,
int opcode, PyObject *dict, PyObject *o)
{
PyObject *mangled = _PyCompile_MaybeMangle(c, o);
if (!mangled) {
return ERROR;
}
Py_ssize_t arg = _PyCompile_DictAddObj(dict, mangled);
Py_DECREF(mangled);
if (arg < 0) {
return ERROR;
}
if (opcode == LOAD_ATTR) {
arg <<= 1;
}
if (opcode == LOAD_METHOD) {
opcode = LOAD_ATTR;
arg <<= 1;
arg |= 1;
}
if (opcode == LOAD_SUPER_ATTR) {
arg <<= 2;
arg |= 2;
}
if (opcode == LOAD_SUPER_METHOD) {
opcode = LOAD_SUPER_ATTR;
arg <<= 2;
arg |= 3;
}
if (opcode == LOAD_ZERO_SUPER_ATTR) {
opcode = LOAD_SUPER_ATTR;
arg <<= 2;
}
if (opcode == LOAD_ZERO_SUPER_METHOD) {
opcode = LOAD_SUPER_ATTR;
arg <<= 2;
arg |= 1;
}
ADDOP_I(c, loc, opcode, arg);
return SUCCESS;
}
#define ADDOP_NAME(C, LOC, OP, O, TYPE) \
RETURN_IF_ERROR(codegen_addop_name((C), (LOC), (OP), METADATA(C)->u_ ## TYPE, (O)))
static int
codegen_addop_j(instr_sequence *seq, location loc,
int opcode, jump_target_label target)
{
assert(IS_JUMP_TARGET_LABEL(target));
assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode));
assert(!IS_ASSEMBLER_OPCODE(opcode));
return _PyInstructionSequence_Addop(seq, opcode, target.id, loc);
}
#define ADDOP_JUMP(C, LOC, OP, O) \
RETURN_IF_ERROR(codegen_addop_j(INSTR_SEQUENCE(C), (LOC), (OP), (O)))
#define ADDOP_COMPARE(C, LOC, CMP) \
RETURN_IF_ERROR(codegen_addcompare((C), (LOC), (cmpop_ty)(CMP)))
#define ADDOP_BINARY(C, LOC, BINOP) \
RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), false))
#define ADDOP_INPLACE(C, LOC, BINOP) \
RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), true))
#define ADD_YIELD_FROM(C, LOC, await) \
RETURN_IF_ERROR(codegen_add_yield_from((C), (LOC), (await)))
#define POP_EXCEPT_AND_RERAISE(C, LOC) \
RETURN_IF_ERROR(codegen_pop_except_and_reraise((C), (LOC)))
#define ADDOP_YIELD(C, LOC) \
RETURN_IF_ERROR(codegen_addop_yield((C), (LOC)))
/* VISIT and VISIT_SEQ takes an ASDL type as their second argument. They use
the ASDL name to synthesize the name of the C type and the visit function.
*/
#define VISIT(C, TYPE, V) \
RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), (V)))
#define VISIT_IN_SCOPE(C, TYPE, V) \
RETURN_IF_ERROR_IN_SCOPE((C), codegen_visit_ ## TYPE((C), (V)))
#define VISIT_SEQ(C, TYPE, SEQ) \
do { \
asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
for (int _i = 0; _i < asdl_seq_LEN(seq); _i++) { \
TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \
RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), elt)); \
} \
} while (0)
#define VISIT_SEQ_IN_SCOPE(C, TYPE, SEQ) \
do { \
asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
for (int _i = 0; _i < asdl_seq_LEN(seq); _i++) { \
TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \
if (codegen_visit_ ## TYPE((C), elt) < 0) { \
_PyCompile_ExitScope(C); \
return ERROR; \
} \
} \
} while (0)
static int
codegen_call_exit_with_nones(compiler *c, location loc)
{
ADDOP_LOAD_CONST(c, loc, Py_None);
ADDOP_LOAD_CONST(c, loc, Py_None);
ADDOP_LOAD_CONST(c, loc, Py_None);
ADDOP_I(c, loc, CALL, 3);
return SUCCESS;
}
static int
codegen_add_yield_from(compiler *c, location loc, int await)
{
NEW_JUMP_TARGET_LABEL(c, send);
NEW_JUMP_TARGET_LABEL(c, fail);
NEW_JUMP_TARGET_LABEL(c, exit);
USE_LABEL(c, send);
ADDOP_JUMP(c, loc, SEND, exit);
// Set up a virtual try/except to handle when StopIteration is raised during
// a close or throw call. The only way YIELD_VALUE raises if they do!
ADDOP_JUMP(c, loc, SETUP_FINALLY, fail);
ADDOP_I(c, loc, YIELD_VALUE, 1);
ADDOP(c, NO_LOCATION, POP_BLOCK);
ADDOP_I(c, loc, RESUME, await ? RESUME_AFTER_AWAIT : RESUME_AFTER_YIELD_FROM);
ADDOP_JUMP(c, loc, JUMP_NO_INTERRUPT, send);
USE_LABEL(c, fail);
ADDOP(c, loc, CLEANUP_THROW);
USE_LABEL(c, exit);
ADDOP(c, loc, END_SEND);
return SUCCESS;
}
static int
codegen_pop_except_and_reraise(compiler *c, location loc)
{
/* Stack contents
* [exc_info, lasti, exc] COPY 3
* [exc_info, lasti, exc, exc_info] POP_EXCEPT
* [exc_info, lasti, exc] RERAISE 1
* (exception_unwind clears the stack)
*/
ADDOP_I(c, loc, COPY, 3);
ADDOP(c, loc, POP_EXCEPT);
ADDOP_I(c, loc, RERAISE, 1);
return SUCCESS;
}
/* Unwind a frame block. If preserve_tos is true, the TOS before
* popping the blocks will be restored afterwards, unless another
* return, break or continue is found. In which case, the TOS will
* be popped.
*/
static int
codegen_unwind_fblock(compiler *c, location *ploc,
fblockinfo *info, int preserve_tos)
{
switch (info->fb_type) {
case COMPILE_FBLOCK_WHILE_LOOP:
case COMPILE_FBLOCK_EXCEPTION_HANDLER:
case COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER:
case COMPILE_FBLOCK_ASYNC_COMPREHENSION_GENERATOR:
case COMPILE_FBLOCK_STOP_ITERATION:
return SUCCESS;
case COMPILE_FBLOCK_FOR_LOOP:
/* Pop the iterator */
if (preserve_tos) {
ADDOP_I(c, *ploc, SWAP, 2);
}
ADDOP(c, *ploc, POP_TOP);
return SUCCESS;
case COMPILE_FBLOCK_TRY_EXCEPT:
ADDOP(c, *ploc, POP_BLOCK);
return SUCCESS;
case COMPILE_FBLOCK_FINALLY_TRY:
/* This POP_BLOCK gets the line number of the unwinding statement */
ADDOP(c, *ploc, POP_BLOCK);
if (preserve_tos) {
RETURN_IF_ERROR(
_PyCompile_PushFBlock(c, *ploc, COMPILE_FBLOCK_POP_VALUE,
NO_LABEL, NO_LABEL, NULL));
}
/* Emit the finally block */
VISIT_SEQ(c, stmt, info->fb_datum);
if (preserve_tos) {
_PyCompile_PopFBlock(c, COMPILE_FBLOCK_POP_VALUE, NO_LABEL);
}
/* The finally block should appear to execute after the
* statement causing the unwinding, so make the unwinding
* instruction artificial */
*ploc = NO_LOCATION;
return SUCCESS;
case COMPILE_FBLOCK_FINALLY_END:
if (preserve_tos) {
ADDOP_I(c, *ploc, SWAP, 2);
}
ADDOP(c, *ploc, POP_TOP); /* exc_value */
if (preserve_tos) {
ADDOP_I(c, *ploc, SWAP, 2);
}
ADDOP(c, *ploc, POP_BLOCK);
ADDOP(c, *ploc, POP_EXCEPT);
return SUCCESS;
case COMPILE_FBLOCK_WITH:
case COMPILE_FBLOCK_ASYNC_WITH:
*ploc = info->fb_loc;
ADDOP(c, *ploc, POP_BLOCK);
if (preserve_tos) {
ADDOP_I(c, *ploc, SWAP, 3);
ADDOP_I(c, *ploc, SWAP, 2);
}
RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc));
if (info->fb_type == COMPILE_FBLOCK_ASYNC_WITH) {
ADDOP_I(c, *ploc, GET_AWAITABLE, 2);
ADDOP_LOAD_CONST(c, *ploc, Py_None);
ADD_YIELD_FROM(c, *ploc, 1);
}
ADDOP(c, *ploc, POP_TOP);
/* The exit block should appear to execute after the
* statement causing the unwinding, so make the unwinding
* instruction artificial */
*ploc = NO_LOCATION;
return SUCCESS;
case COMPILE_FBLOCK_HANDLER_CLEANUP: {
if (info->fb_datum) {
ADDOP(c, *ploc, POP_BLOCK);
}
if (preserve_tos) {
ADDOP_I(c, *ploc, SWAP, 2);
}
ADDOP(c, *ploc, POP_BLOCK);
ADDOP(c, *ploc, POP_EXCEPT);
if (info->fb_datum) {
ADDOP_LOAD_CONST(c, *ploc, Py_None);
RETURN_IF_ERROR(codegen_nameop(c, *ploc, info->fb_datum, Store));
RETURN_IF_ERROR(codegen_nameop(c, *ploc, info->fb_datum, Del));
}
return SUCCESS;
}
case COMPILE_FBLOCK_POP_VALUE: {
if (preserve_tos) {
ADDOP_I(c, *ploc, SWAP, 2);
}
ADDOP(c, *ploc, POP_TOP);
return SUCCESS;
}
}
Py_UNREACHABLE();
}
/** Unwind block stack. If loop is not NULL, then stop when the first loop is encountered. */
static int
codegen_unwind_fblock_stack(compiler *c, location *ploc,
int preserve_tos, fblockinfo **loop)
{
fblockinfo *top = _PyCompile_TopFBlock(c);
if (top == NULL) {
return SUCCESS;
}
if (top->fb_type == COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER) {
return _PyCompile_Error(
c, *ploc, "'break', 'continue' and 'return' cannot appear in an except* block");
}
if (loop != NULL && (top->fb_type == COMPILE_FBLOCK_WHILE_LOOP ||
top->fb_type == COMPILE_FBLOCK_FOR_LOOP)) {
*loop = top;
return SUCCESS;
}
fblockinfo copy = *top;
_PyCompile_PopFBlock(c, top->fb_type, top->fb_block);
RETURN_IF_ERROR(codegen_unwind_fblock(c, ploc, ©, preserve_tos));
RETURN_IF_ERROR(codegen_unwind_fblock_stack(c, ploc, preserve_tos, loop));
_PyCompile_PushFBlock(c, copy.fb_loc, copy.fb_type, copy.fb_block,
copy.fb_exit, copy.fb_datum);
return SUCCESS;
}
static int
codegen_enter_scope(compiler *c, identifier name, int scope_type,
void *key, int lineno, PyObject *private,
_PyCompile_CodeUnitMetadata *umd)
{
RETURN_IF_ERROR(
_PyCompile_EnterScope(c, name, scope_type, key, lineno, private, umd));
location loc = LOCATION(lineno, lineno, 0, 0);
if (scope_type == COMPILE_SCOPE_MODULE) {
loc.lineno = 0;
}
ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START);
return SUCCESS;
}
static int
codegen_setup_annotations_scope(compiler *c, location loc,
void *key, PyObject *name)
{
_PyCompile_CodeUnitMetadata umd = {
.u_posonlyargcount = 1,
};
RETURN_IF_ERROR(
codegen_enter_scope(c, name, COMPILE_SCOPE_ANNOTATIONS,
key, loc.lineno, NULL, &umd));
// if .format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError
PyObject *value_with_fake_globals = PyLong_FromLong(_Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS);
assert(!SYMTABLE_ENTRY(c)->ste_has_docstring);
_Py_DECLARE_STR(format, ".format");
ADDOP_I(c, loc, LOAD_FAST, 0);
ADDOP_LOAD_CONST(c, loc, value_with_fake_globals);
ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]);
NEW_JUMP_TARGET_LABEL(c, body);
ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body);
ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR);
ADDOP_I(c, loc, RAISE_VARARGS, 1);
USE_LABEL(c, body);
return SUCCESS;
}
static int
codegen_leave_annotations_scope(compiler *c, location loc)
{
ADDOP_IN_SCOPE(c, loc, RETURN_VALUE);
PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1);
if (co == NULL) {
return ERROR;
}
// We want the parameter to __annotate__ to be named "format" in the
// signature shown by inspect.signature(), but we need to use a
// different name (.format) in the symtable; if the name
// "format" appears in the annotations, it doesn't get clobbered
// by this name. This code is essentially:
// co->co_localsplusnames = ("format", *co->co_localsplusnames[1:])
const Py_ssize_t size = PyObject_Size(co->co_localsplusnames);
if (size == -1) {
Py_DECREF(co);
return ERROR;
}
PyObject *new_names = PyTuple_New(size);
if (new_names == NULL) {
Py_DECREF(co);
return ERROR;
}
PyTuple_SET_ITEM(new_names, 0, Py_NewRef(&_Py_ID(format)));
for (int i = 1; i < size; i++) {
PyObject *item = PyTuple_GetItem(co->co_localsplusnames, i);
if (item == NULL) {
Py_DECREF(co);
Py_DECREF(new_names);
return ERROR;
}
Py_INCREF(item);
PyTuple_SET_ITEM(new_names, i, item);
}
Py_SETREF(co->co_localsplusnames, new_names);
_PyCompile_ExitScope(c);
int ret = codegen_make_closure(c, loc, co, 0);
Py_DECREF(co);
RETURN_IF_ERROR(ret);
return SUCCESS;
}
static int
codegen_deferred_annotations_body(compiler *c, location loc,
PyObject *deferred_anno, PyObject *conditional_annotation_indices, int scope_type)
{
Py_ssize_t annotations_len = PyList_GET_SIZE(deferred_anno);
assert(PyList_CheckExact(conditional_annotation_indices));
assert(annotations_len == PyList_Size(conditional_annotation_indices));
ADDOP_I(c, loc, BUILD_MAP, 0); // stack now contains <annos>
for (Py_ssize_t i = 0; i < annotations_len; i++) {
PyObject *ptr = PyList_GET_ITEM(deferred_anno, i);
stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr);
if (st == NULL) {
return ERROR;
}
PyObject *mangled = _PyCompile_Mangle(c, st->v.AnnAssign.target->v.Name.id);
if (!mangled) {
return ERROR;
}
PyObject *cond_index = PyList_GET_ITEM(conditional_annotation_indices, i);
assert(PyLong_CheckExact(cond_index));
long idx = PyLong_AS_LONG(cond_index);
NEW_JUMP_TARGET_LABEL(c, not_set);
if (idx != -1) {
ADDOP_LOAD_CONST(c, LOC(st), cond_index);
if (scope_type == COMPILE_SCOPE_CLASS) {
ADDOP_NAME(
c, LOC(st), LOAD_DEREF, &_Py_ID(__conditional_annotations__), freevars);
}
else {
ADDOP_NAME(
c, LOC(st), LOAD_GLOBAL, &_Py_ID(__conditional_annotations__), names);
}
ADDOP_I(c, LOC(st), CONTAINS_OP, 0);
ADDOP_JUMP(c, LOC(st), POP_JUMP_IF_FALSE, not_set);
}
VISIT(c, expr, st->v.AnnAssign.annotation);
ADDOP_I(c, LOC(st), COPY, 2);
ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled);
// stack now contains <annos> <name> <annos> <value>
ADDOP(c, loc, STORE_SUBSCR);
// stack now contains <annos>
USE_LABEL(c, not_set);
}
return SUCCESS;
}
static int
codegen_process_deferred_annotations(compiler *c, location loc)
{
PyObject *deferred_anno = NULL;
PyObject *conditional_annotation_indices = NULL;
_PyCompile_DeferredAnnotations(c, &deferred_anno, &conditional_annotation_indices);
if (deferred_anno == NULL) {
assert(conditional_annotation_indices == NULL);
return SUCCESS;
}
// It's possible that ste_annotations_block is set but
// u_deferred_annotations is not, because the former is still
// set if there are only non-simple annotations (i.e., annotations
// for attributes, subscripts, or parenthesized names). However, the
// reverse should not be possible.
PySTEntryObject *ste = SYMTABLE_ENTRY(c);
assert(ste->ste_annotation_block != NULL);
void *key = (void *)((uintptr_t)ste->ste_id + 1);
int scope_type = SCOPE_TYPE(c);
if (codegen_setup_annotations_scope(c, loc, key,
ste->ste_annotation_block->ste_name) < 0) {
goto error;
}
if (codegen_deferred_annotations_body(c, loc, deferred_anno,
conditional_annotation_indices, scope_type) < 0) {
_PyCompile_ExitScope(c);
goto error;
}
Py_DECREF(deferred_anno);
Py_DECREF(conditional_annotation_indices);
RETURN_IF_ERROR(codegen_leave_annotations_scope(c, loc));
RETURN_IF_ERROR(codegen_nameop(c, loc, &_Py_ID(__annotate__), Store));
return SUCCESS;
error:
Py_XDECREF(deferred_anno);
Py_XDECREF(conditional_annotation_indices);
return ERROR;
}
/* Compile an expression */
int
_PyCodegen_Expression(compiler *c, expr_ty e)
{
VISIT(c, expr, e);
return SUCCESS;
}
/* Compile a sequence of statements, checking for a docstring
and for annotations. */
int
_PyCodegen_Module(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interactive)
{
if (SYMTABLE_ENTRY(c)->ste_has_conditional_annotations) {
ADDOP_I(c, loc, BUILD_SET, 0);
ADDOP_N(c, loc, STORE_NAME, &_Py_ID(__conditional_annotations__), names);
}
return codegen_body(c, loc, stmts, is_interactive);
}
int
codegen_body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interactive)
{
/* If from __future__ import annotations is active,
* every annotated class and module should have __annotations__.
* Else __annotate__ is created when necessary. */
PySTEntryObject *ste = SYMTABLE_ENTRY(c);
if ((FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && ste->ste_annotations_used) {
ADDOP(c, loc, SETUP_ANNOTATIONS);
}
if (!asdl_seq_LEN(stmts)) {
return SUCCESS;
}
Py_ssize_t first_instr = 0;
if (!is_interactive) { /* A string literal on REPL prompt is not a docstring */
if (ste->ste_has_docstring) {
PyObject *docstring = _PyAST_GetDocString(stmts);
assert(docstring);
first_instr = 1;
/* set docstring */
assert(OPTIMIZATION_LEVEL(c) < 2);
PyObject *cleandoc = _PyCompile_CleanDoc(docstring);
if (cleandoc == NULL) {
return ERROR;
}
stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
assert(st->kind == Expr_kind);
location loc = LOC(st->v.Expr.value);
ADDOP_LOAD_CONST(c, loc, cleandoc);
Py_DECREF(cleandoc);
RETURN_IF_ERROR(codegen_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store));
}
}
for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) {
VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
}
// If there are annotations and the future import is not on, we
// collect the annotations in a separate pass and generate an
// __annotate__ function. See PEP 649.
if (!(FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS)) {
RETURN_IF_ERROR(codegen_process_deferred_annotations(c, loc));
}
return SUCCESS;
}
int
_PyCodegen_EnterAnonymousScope(compiler* c, mod_ty mod)
{
_Py_DECLARE_STR(anon_module, "<module>");
RETURN_IF_ERROR(
codegen_enter_scope(c, &_Py_STR(anon_module), COMPILE_SCOPE_MODULE,
mod, 1, NULL, NULL));
return SUCCESS;
}
static int
codegen_make_closure(compiler *c, location loc,
PyCodeObject *co, Py_ssize_t flags)
{
if (co->co_nfreevars) {
int i = PyUnstable_Code_GetFirstFree(co);
for (; i < co->co_nlocalsplus; ++i) {
/* Bypass com_addop_varname because it will generate
LOAD_DEREF but LOAD_CLOSURE is needed.
*/
PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
int arg = _PyCompile_LookupArg(c, co, name);
RETURN_IF_ERROR(arg);
ADDOP_I(c, loc, LOAD_CLOSURE, arg);
}
flags |= MAKE_FUNCTION_CLOSURE;
ADDOP_I(c, loc, BUILD_TUPLE, co->co_nfreevars);
}
ADDOP_LOAD_CONST(c, loc, (PyObject*)co);
ADDOP(c, loc, MAKE_FUNCTION);
if (flags & MAKE_FUNCTION_CLOSURE) {
ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_CLOSURE);
}
if (flags & MAKE_FUNCTION_ANNOTATIONS) {
ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS);
}
if (flags & MAKE_FUNCTION_ANNOTATE) {
ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE);
}
if (flags & MAKE_FUNCTION_KWDEFAULTS) {
ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS);
}
if (flags & MAKE_FUNCTION_DEFAULTS) {
ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_DEFAULTS);
}
return SUCCESS;
}
static int
codegen_decorators(compiler *c, asdl_expr_seq* decos)
{
if (!decos) {
return SUCCESS;
}
for (Py_ssize_t i = 0; i < asdl_seq_LEN(decos); i++) {
VISIT(c, expr, (expr_ty)asdl_seq_GET(decos, i));
}
return SUCCESS;
}
static int
codegen_apply_decorators(compiler *c, asdl_expr_seq* decos)
{
if (!decos) {
return SUCCESS;
}
for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) {
location loc = LOC((expr_ty)asdl_seq_GET(decos, i));
ADDOP_I(c, loc, CALL, 0);
}
return SUCCESS;
}
static int
codegen_kwonlydefaults(compiler *c, location loc,
asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults)
{
/* Push a dict of keyword-only default values.
Return -1 on error, 0 if no dict pushed, 1 if a dict is pushed.
*/
int default_count = 0;
for (int i = 0; i < asdl_seq_LEN(kwonlyargs); i++) {
arg_ty arg = asdl_seq_GET(kwonlyargs, i);
expr_ty default_ = asdl_seq_GET(kw_defaults, i);
if (default_) {
default_count++;
PyObject *mangled = _PyCompile_MaybeMangle(c, arg->arg);
if (!mangled) {
return ERROR;
}
ADDOP_LOAD_CONST_NEW(c, loc, mangled);
VISIT(c, expr, default_);
}
}
if (default_count) {
ADDOP_I(c, loc, BUILD_MAP, default_count);
return 1;
}
else {
return 0;
}
}