From 1cbb0d6474f516a08d9152b17007e5cf713b6a2c Mon Sep 17 00:00:00 2001 From: Erik Stenman Date: Mon, 27 Mar 2017 00:55:43 +0200 Subject: [PATCH] Adding some more chapters and examples. --- .gitignore | 4 +- Makefile | 19 +- ap-beam_instructions.asciidoc | 742 ++++++++++ beam_instructions.asciidoc | 503 +++++++ beam_internal_instructions.asciidoc | 4 + book.asciidoc | 14 +- calls.asciidoc | 149 ++ code/book/src/generate_op_doc.erl | 120 ++ code/compiler_chapter/src/json_tokens.xrl | 68 + code/compiler_chapter/src/world.E | 17 + code/compiler_chapter/src/world.P | 15 + code/compiler_chapter/src/world.S | 37 + code/compiler_chapter/src/world.erl | 7 + code/compiler_chapter/src/world.hrl | 1 + code/memory_chapter/src/gc_example.erl | 19 + compiler.asciidoc | 1517 ++++++++++----------- genop.tab | 539 ++++++++ memory.asciidoc | 1410 +++++++++++++++++++ preface.asciidoc | 4 +- type_system.asciidoc | 354 +++++ 20 files changed, 4765 insertions(+), 778 deletions(-) create mode 100755 ap-beam_instructions.asciidoc create mode 100755 beam_instructions.asciidoc create mode 100755 beam_internal_instructions.asciidoc create mode 100755 calls.asciidoc create mode 100755 code/book/src/generate_op_doc.erl create mode 100644 code/compiler_chapter/src/json_tokens.xrl create mode 100644 code/compiler_chapter/src/world.E create mode 100644 code/compiler_chapter/src/world.P create mode 100644 code/compiler_chapter/src/world.S create mode 100644 code/compiler_chapter/src/world.erl create mode 100644 code/compiler_chapter/src/world.hrl create mode 100644 code/memory_chapter/src/gc_example.erl create mode 100755 genop.tab create mode 100644 memory.asciidoc create mode 100755 type_system.asciidoc diff --git a/.gitignore b/.gitignore index 90e89af..65a8b56 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ rel/example_project git-log.xml book-revhistory.xml beam-book-from-ab.xml -beam-book.pdf \ No newline at end of file +beam-book.pdf +opcodes_doc.asciidoc +.#.gitignore diff --git a/Makefile b/Makefile index 5ad8f0b..337a206 100755 --- a/Makefile +++ b/Makefile @@ -4,14 +4,16 @@ adocs = book.asciidoc \ preface.asciidoc \ introduction.asciidoc \ compiler.asciidoc \ - processes.asciidoc + processes.asciidoc \ + calls.asciidoc \ + type_system.asciidoc \ + beam_internal_instructions.asciidoc \ + memory.asciidoc \ + ap-beam_instructions.asciidoc \ + opcodes_doc.asciidoc # beam.asciidoc \ # beam_modules.asciidoc \ - # type_system.asciidoc \ - # memory.asciidoc \ # erts-book.asciidoc \ - # ap-beam_instructions.asciidoc \ - # opcodes_doc.asciidoc all: beam-book.pdf @@ -33,6 +35,13 @@ beam-book.pdf: beam-book-from-ab.xml book-revhistory.xml # index.html: *.asciidoc # asciidoc -o index.html -a icons -a toc2 book.asciidoc +code/book/ebin/generate_op_doc.beam: code/book/src/generate_op_doc.erl + erlc -o $@ $< + +opcodes_doc.asciidoc: genop.tab code/book/ebin/generate_op_doc.beam + erl -noshell -s generate_op_doc from_shell genop.tab opcodes_doc.asciidoc + + # generate_op_doc.beam: generate_op_doc.erl # erlc generate_op_doc.erl diff --git a/ap-beam_instructions.asciidoc b/ap-beam_instructions.asciidoc new file mode 100755 index 0000000..09d769d --- /dev/null +++ b/ap-beam_instructions.asciidoc @@ -0,0 +1,742 @@ +[[AP-Instructions]] +== Appendix A: BEAM Instructions + +Here we will go through most of the instructions in the BEAM +generic instruction set in detail. In the next section we list +all instructions with a brief explanation generated from the +documentaion in the code (see +lib/compiler/src/genop.tab+). + +=== Functions and Labels + +==== label Lbl + +Instruction number 1 in the generic instruction set is not really an +instruction at all. It is just a module local label giving a name, or +actually a number to the current position in the code. + +Each label potentially marks the beginning of a basic block since +it is a potential destination of a jump. + +==== func_info Module Function Arity + +The code for each function starts with a func_info instruction. +This instruction is used for generating a function clause error, +and the execution of the code in the function actually starts +at the label following the func_info instruction. + +Imagine a function with a guard: + +[source,erlang] +------------------------------------------ +id(I) when is_integer(I) -> I. +------------------------------------------ + +The Beam code for this function might look like: + +[source,erlang] +------------------------------------------ + +{function, id, 1, 4}. + {label,3}. + {func_info,{atom,test1},{atom,id},1}. + {label,4}. + {test,is_integer,{f,3},[{x,0}]}. + return. + +------------------------------------------ + +Here the meta information +{function, id, 1, 4}+ tells us that +execution of the id/1 function will start at label 4. At label 4 we do +an +is_integer+ on x0 and if we fail we jump to label 3 (f3) which +points to the func_info instruction, which will generate a _function +clause_ exception. Otherwise we just fall through and return the +argument (x0). + +=== Test instructions + +==== Type tests + +The type test instructions (+is_\* Lbl Argument+) checks whether the +argument is of the given type and if not jumps to the label Lbl. +The beam disassembler wraps all these instructions in a +test+ +instruction. E.g.: + +[source,erlang] +------------------------------------------ + {test,is_integer,{f,3},[{x,0}]}. +------------------------------------------ + +The current type test instructions are is_integer, is_float, +is_number, is_atom, is_pid, is_reference, is_port, is_nil, is_binary, +is_list, is_nonempty_list, is_function, is_function2, is_boolean, +is_bitstr, and is_tuple. + +And then there is also one type test instruction of Arity 3: ++test_arity Lbl Arg Arity+. This instruction tests that +the arity of the argument (assumed to be a tuple) is of +Arity+. +This instruction is usually preceded by an +is_tuple+ +instruction. + +==== Comparisons + +The comparison instructions (+is_\* Lbl Arg1 Arg2+) compares the +two arguments according to the instructions and jumps to Lbl if the +comparison fails. + +The comparison instructions are: : is_lt, is_ge, is_eq, is_ne, +is_eq_exact, and is_ne_exact. + +Remember that all Erlang terms are ordered so these instructions can +compare any two terms. You can for example test if the atom +self+ +is less than the pid returned by +self()+. (It is.) + +Note that for numbers the comparison is done on the Erlang type +_number_, see xref:CH-TypeSystem[]. That is, for a mixed float and +integer comparison the number of lower precision is converted to the +other type before comparison. For example on my system 1 and 0.1 +compares as equal, as well as 9999999999999999 and 1.0e16. +Comparing floating point numbers is always risk and best avoided, +the result may wary depending on the underlying hardware. + +If you want to make sure that the integer 1 and the floating point +number 1.0 are compared different you can use is_eq_exact and +is_ne_exact. This corresponds to the Erlang operators +=:=+ and ++=/=+. + +=== Function Calls + +In this chapter we will summarize what the different call instructions +does. For a thorough description of how function calls work see +xref:CH-Calls[]. + +==== call Arity Label + +Does a call to the function of arity +Arity+ in the same +module at label +Label+. First count down the reductions and +if needed do a context switch. + +For all local calls the label is the second label of the +function where the code starts. It is assumed that the preceding +instruction at that label is +func_info+ in order to get the MFA if a +context switch is needed. + +==== call_last Arity Label + +Do a tail recursive call the function of arity +Arity+ in the same +module at label +Label+. First count down the reductions and if needed +do a context switch. + + +==== call_only Arity Label Deallocate + +Deallocate +Deallocate+ words of stack, then do a tail recursive call +to the function of arity +Arity+ in the same module at label +Label+ First +count down the reductions and if needed do a context switch. + +==== call_ext Arity Destination + +Does an external call to the function of arity +Arity+ given by + Destination. Destination is usually of the form +{extfunc, Module, + Function, Arity}+. First count down the reductions and if needed do a + context switch. + +==== call_ext_only Arity Destination + +Does a tail recursive external call to the function of arity +Arity+ given by + Destination. Destination is usually of the form +{extfunc, Module, + Function, Arity}+. First count down the reductions and if needed do a + context switch. + + +==== call_ext_last Arity Destination Deallocate + +Deallocate +Deallocate+ words of stack, then do a tail recursive +external call to the function of arity +Arity+ given by +Destination. Destination is usually of the form +{extfunc, Module, +Function, Arity}+. First count down the reductions and if needed do a +context switch. + + +==== +bif0 Bif Reg+, +bif1 Lbl Bif Arg Reg+, +bif2 Lbl Bif Arg1 Arg2 Reg+ + +Call the bif +Bif+ with the given arguments, and store the result in ++Reg+. If the bif fails jump to +Lbl+. No zero arity bif can fail an +thus those calls doesn't take a fail label. + +// Bif called by these instructions may not allocate on the heap nor +// trigger a garbage collection. + +==== +gc_bif1-3 Lbl Live Bif Arg1-3 Reg+ + +Call the bif +Bif+ with the given arguments, and store the result in Reg. +If the bif fails jump to Lbl. +Store the arguments in x(Live), x(Live+1) and x(live+2). + +==== +call_fun Arity+ + +The instruction +call_fun+ assumes that the arguments are placed in +the argument registers and that the fun (the pointer to the closure) +is placed in the last argument register. + +That is, for a zero arity call, the closure is in x0. For a arity 1 call +x0 contains the argument and x1 contains the closure. + +==== apply/1 + +TODO + +==== apply_last/2 + +TODO + +=== Stack (and Heap) Management + +The stack and the heap of an Erlang process on Beam share the same memory +area see xref:CH-Processes[] and xref:CH-Memory[] for a full discussion. +The stack grows toward lower addresses and the heap toward higher addresses. +Beam will do a garbage collection if more space than what is available is +needed on either the stack or the heap. + +************************** + +*A leaf function*:: A leaf function is a function which doesn't call + any other function. + +*A non leaf function*:: A non leaf function is a function which may call + another function. + +************************** + + +These instructions are also used by non leaf functions for setting up +and tearing down the stack frame for the current instruction. That is, +on entry to the function the _continuation pointer_ (CP) is saved on +the stack, and on exit it is read back from the stack. + +A function skeleton for a leaf function looks like this: + +---- +{function, Name, Arity, StartLabel}. + {label,L1}. + {func_info,{atom,Module},{atom,Name},Arity}. + {label,L2}. + ... + return. +---- + + +A function skeleton for a non leaf function looks like this: + +---- +{function, Name, Arity, StartLabel}. + {label,L1}. + {func_info,{atom,Module},{atom,Name},Arity}. + {label,L2}. + {allocate,Need,Live}. + + ... + call ... + ... + + {deallocate,Need}. + return. +---- + + + +==== +allocate StackNeed Live+ + +Save the continuation pointer (CP) and allocate space for +StackNeed+ +extra words on the stack. If a GC is needed during allocation save ++Live+ number of X registers. E.g. if +Live+ is 2 then registers X0 +and X1 are saved. + +When allocating on the stack, the stack pointer (E) is decreased. + +.Allocate 1 0 +==== +---- + Before After + | xxx | | xxx | + E -> | xxx | | xxx | + | | | ??? | caller save slot + ... E -> | CP | + ... ... + HTOP -> | | HTOP -> | | + | xxx | | xxx | +---- +==== + +==== +allocate_heap StackNeed HeapNeed Live+ + +Save the continuation pointer (CP) and allocate space for +StackNeed+ +extra words on the stack. Ensure that there also is space for HeapNeed +words on the heap. If a GC is needed during allocation save +Live+ +number of X registers. + +Note that the heap pointer (HTOP) is not changed until the actual heap +allocation takes place. + +==== +allocate_zero StackNeed Live+ + +This instruction works the same way as allocate, but it also clears +out the allocated stack slots with NIL. + +.allocate_zero 1 0 +==== +---- + Before After + | xxx | | xxx | + E -> | xxx | | xxx | + | | | NIL | caller save slot + ... E -> | CP | + ... ... + HTOP -> | | HTOP -> | | + | xxx | | xxx | +---- +==== + +==== +allocate_heap_zero StackNeed HeapNeed Live+ + +The allocate_heap_zero instruction works as the allocate_heap +instruction, but it also clears out the allocated stack slots +with NIL. + +==== +test_heap HeapNeed Live+ + +The test_heap instruction ensures there is space for HeapNeed words on +the heap. If a GC is needed save Live number of X registers. + +==== +init N+ + +The init instruction clears N stack words. By writing NIL to them. + +==== +deallocate N+ + +The deallocate instruction is the opposite of the allocate instruction, +it restores the continuation pointer and deallocates N+1 stack words. + +==== +return+ +The return instructions jumps to the address in the continuation pointer (CP). + +==== +trim N Remaining+ +Removes N words of stack usage, while keeping the continuation pointer +on the top of the stack. (The argument Remaining is to the best of my +knowledge unused.) + +---- +Trim 2 + Before After + | ??? | | ??? | + | xxx | E -> | CP | + | xxx | | ... | + E -> | CP | | ... | + | | | ... | + ... ... + HTOP -> | | HTOP -> | | + | xxx | | xxx | +---- + + +=== Moving, extracting, modifying. +==== move Source Destination + +Move the source Source (a litteral or a register) to the destination +register Destination. + +==== get_list Source Head Tail + +Get the head and tail (or car and cdr) parts of a list (a cons cell) from +Source and put them into the registers Head and Tail. + +==== get_tuple_element Source Element Destination + +Get element number Element from the tuple in Source and put it in the +destination register Destination. + +==== set_tuple_element NewElement Tuple Position + +Update the element at postition Position of the tuple Tuple with the +new element NewElement. + +=== Building terms. + +==== put_list/3 +TODO +==== put_tuple/2 +TODO +==== put/1 + +==== make_fun2/1 +TODO + +=== Binary Syntax +==== bs_put_integer/5 +TODO +==== bs_put_binary/5 +TODO +==== bs_put_float/5 +TODO +==== bs_put_string/2 +TODO +==== bs_init2/6 +TODO +==== bs_add/5 +TODO +==== bs_start_match2/5 +TODO +==== bs_get_integer2/7 +TODO +==== bs_get_float2/7 +TODO +==== bs_get_binary2/7 +TODO +==== bs_skip_bits2/5 +TODO +==== bs_test_tail2/3 +TODO +==== bs_save2/2 +TODO +==== bs_restore2/2 +TODO +==== bs_context_to_binary/1 +TODO +==== bs_test_unit/3 +TODO +==== bs_match_string/4 +TODO +==== bs_init_writable/0 +TODO +==== bs_append/8 +TODO +==== bs_private_append/6 +TODO +==== bs_init_bits/6 +TODO +==== bs_get_utf8/5 +TODO +==== bs_skip_utf8/4 +TODO +==== bs_get_utf16/5 +TODO +==== bs_skip_utf16/4 +TODO +==== bs_get_utf32/5 +TODO +==== bs_skip_utf32/4 +TODO +==== bs_utf8_size/3 +TODO +==== bs_put_utf8/3 +TODO +==== bs_utf16_size/3 +TODO +==== bs_put_utf16/3 +TODO +==== bs_put_utf32/3 +TODO + +=== Floating Point Arithmetic +==== fclearerror/0 +TODO +==== fcheckerror/1 +TODO +==== fmove/2 +TODO +==== fconv/2 +TODO +==== fadd/4 +TODO +==== fsub/4 +TODO +==== fmul/4 +TODO +==== fdiv/4 +TODO +==== fnegate/3 +TODO + + +=== Pattern Matching + +==== select_val +TODO +==== select_arity_val +TODO +==== jump +TODO +=== Exception handling +==== catch/2 +TODO +==== catch_end/1 +TODO +==== badmatch/1 +TODO +==== if_end/0 +TODO +==== case_end/1 +TODO + +=== Meta instructions +==== on_load +TODO +==== line +TODO + + +include::opcodes_doc.asciidoc[] + +=== Specific Instructions + +Argument types +[options="header"] +|================================================== +|Type | Explanation +|t| A term, e.g. +[{foo,bar}]+ +|I| An integer e.g. +42+ +|x| A register, e.g. +5+ +|y| A stack slot, e.g. +1+ +|c| A constant (atom,nil,small int) // Pid? +|a| An atom, e.g. 'foo' +|f| A label, i.e. a code address +|s| Either a literal, a register or a stack slot +|d| Either a register or a stack slot +|r| A register R0 +|P| A positive (unsigned) integer literal +|j| An optional code label +|e| A reference to an export table entry +|l| A floating-point register +|================================================== + +List of all BEAM Instructions + +[options="header"] +|=========== +|Instruction | Arguments | Explanation +|allocate | t t | +|allocate_heap |t I t | +|deallocate |I | +|init |y | +|init2 |y y | +|init3 |y y y | +|i_trim |I | +|test_heap |I t | +|allocate_zero |t t | +|allocate_heap_zero | t I t | +|i_select_val | r f I | +|i_select_val | x f I | +|i_select_val | y f I | +|i_select_val2 | r f c f c f | +|i_select_val2 | x f c f c f | +|i_select_val2 | y f c f c f | +|i_jump_on_val | rxy f I I | +|i_jump_on_val_zero | rxy f I | +|i_select_tuple_arity | r f I | +|i_select_tuple_arity | x f I | +|i_select_tuple_arity | y f I | +|i_select_tuple_arity2 | r f A f A f | +|i_select_tuple_arity2 | x f A f A f | +|i_select_tuple_arity2 | y f A f A f | +|i_func_info | I a a I | +|return | | +|get_list | rxy rxy rxy | +|catch | y f | +|catch_end | y | +|try_end | y | +|try_case_end | s | +|set_tuple_element | s d P | +|i_get_tuple_element | rxy P rxy | +|is_number | f rxy | +|jump | f | +|case_end | rxy | +|badmatch | rxy | +|if_end | | +|raise | s s | +|badarg | j | +|system_limit | j | +|move_jump | f ncxy | +|move_x1 | c | +|move_x2 | c | +|move2 | x y x y | +|move2 | y x y x | +|move2 | x x x x | +|move | rxync rxy | +|recv_mark | f | +|i_recv_set | f | +|remove_message | | +|timeout | | +|timeout_locked | | +|i_loop_rec | f r | +|loop_rec_end | f | +|wait | f | +|wait_locked | f | +|wait_unlocked | f | +|i_wait_timeout | f I | +|i_wait_timeout | f s | +|i_wait_timeout_locked | f I | +|i_wait_timeout_locked | f s | +|i_wait_error | | +|i_wait_error_locked | | +|send | | +|i_is_eq_exact_immed | f rxy c | +|i_is_ne_exact_immed | f rxy c | +|i_is_eq_exact_literal | f rxy c | +|i_is_ne_exact_literal | f rxy c | +|i_is_eq_exact | f | +|i_is_ne_exact | f | +|i_is_lt | f | +|i_is_ge | f | +|i_is_eq | f | +|i_is_ne | f | +|i_put_tuple | rxy I | +|put_list | s s d | +|i_fetch | s s | +|move_return | xcn r | +|move_deallocate_return | xycn r Q | +|deallocate_return | Q | +|test_heap_1_put_list | I y | +|is_tuple_of_arity | f rxy A | +|is_tuple | f rxy | +|test_arity | f rxy A | +|extract_next_element | xy | +|extract_next_element2 | xy | +|extract_next_element3 | xy | +|is_integer_allocate | f rx I I | +|is_integer | f rxy | +|is_list | f rxy | +|is_nonempty_list_allocate | f rx I t | +|is_nonempty_list_test_heap | f r I t | +|is_nonempty_list | f rxy | +|is_atom | f rxy | +|is_float | f rxy | +|is_nil | f rxy | +|is_bitstring | f rxy | +|is_reference | f rxy | +|is_pid | f rxy | +|is_port | f rxy | +|is_boolean | f rxy | +|is_function2 | f s s | +|allocate_init | t I y | +|i_apply | | +|i_apply_last | P | +|i_apply_only | | +|apply | I | +|apply_last | I P | +|i_apply_fun | | +|i_apply_fun_last | P | +|i_apply_fun_only | | +|i_hibernate | | +|call_bif0 | e | +|call_bif1 | e | +|call_bif2 | e | +|call_bif3 | e | +|i_get | s d | +|self | rxy | +|node | rxy | +|i_fast_element | rxy j I d | +|i_element | rxy j s d | +|bif1 | f b s d | +|bif1_body | b s d | +|i_bif2 | f b d | +|i_bif2_body | b d | +|i_move_call | c r f | +|i_move_call_last | f P c r | +|i_move_call_only | f c r | +|move_call | xy r f | +|move_call_last | xy r f Q | +|move_call_only | x r f | +|i_call | f | +|i_call_last | f P | +|i_call_only | f | +|i_call_ext | e | +|i_call_ext_last | e P | +|i_call_ext_only | e | +|i_move_call_ext | c r e | +|i_move_call_ext_last | e P c r | +|i_move_call_ext_only | e c r | +|i_call_fun | I | +|i_call_fun_last | I P | +|i_make_fun | I t | +|is_function | f rxy | +|i_bs_start_match2 | rxy f I I d | +|i_bs_save2 | rx I | +|i_bs_restore2 | rx I | +|i_bs_match_string | rx f I I | +|i_bs_get_integer_small_imm | rx I f I d | +|i_bs_get_integer_imm | rx I I f I d | +|i_bs_get_integer | f I I d | +|i_bs_get_integer_8 | rx f d | +|i_bs_get_integer_16 | rx f d | +|i_bs_get_integer_32 | rx f I d | +|i_bs_get_binary_imm2 | f rx I I I d | +|i_bs_get_binary2 | f rx I s I d | +|i_bs_get_binary_all2 | f rx I I d | +|i_bs_get_binary_all_reuse | rx f I | +|i_bs_get_float2 | f rx I s I d | +|i_bs_skip_bits2_imm2 | f rx I | +|i_bs_skip_bits2 | f rx rxy I | +|i_bs_skip_bits_all2 | f rx I | +|bs_test_zero_tail2 | f rx | +|bs_test_tail_imm2 | f rx I | +|bs_test_unit | f rx I | +|bs_test_unit8 | f rx | +|bs_context_to_binary | rxy | +|i_bs_get_utf8 | rx f d | +|i_bs_get_utf16 | rx f I d | +|i_bs_validate_unicode_retract | j | +|i_bs_init_fail | rxy j I d | +|i_bs_init_fail_heap | I j I d | +|i_bs_init | I I d | +|i_bs_init_heap | I I I d | +|i_bs_init_heap_bin | I I d | +|i_bs_init_heap_bin_heap | I I I d | +|i_bs_init_bits_fail | rxy j I d | +|i_bs_init_bits_fail_heap | I j I d | +|i_bs_init_bits | I I d | +|i_bs_init_bits_heap | I I I d | +|i_bs_add | j I d | +|i_bs_init_writable | | +|i_bs_append | j I I I d | +|i_bs_private_append | j I d | +|i_new_bs_put_integer | j s I s | +|i_new_bs_put_integer_imm | j I I s | +|i_bs_utf8_size | s d | +|i_bs_utf16_size | s d | +|i_bs_put_utf8 | j s | +|i_bs_put_utf16 | j I s | +|i_bs_validate_unicode | j s | +|i_new_bs_put_float | j s I s | +|i_new_bs_put_float_imm | j I I s | +|i_new_bs_put_binary | j s I s | +|i_new_bs_put_binary_imm | j I s | +|i_new_bs_put_binary_all | j s I | +|bs_put_string | I I | +|fmove | qdl ld | +|fconv | d l | +|i_fadd | l l l | +|i_fsub | l l l | +|i_fmul | l l l | +|i_fdiv | l l l | +|i_fnegate | l l l | +|i_fcheckerror | | +|fclearerror | | +|i_increment | rxy I I d | +|i_plus | j I d | +|i_minus | j I d | +|i_times | j I d | +|i_m_div | j I d | +|i_int_div | j I d | +|i_rem | j I d | +|i_bsl | j I d | +|i_bsr | j I d | +|i_band | j I d | +|i_bor | j I d | +|i_bxor | j I d | +|i_int_bnot | j s I d | +|i_gc_bif1 | j I s I d | +|i_gc_bif2 | j I I d | +|i_gc_bif3 | j I s I d | +|int_code_end | | +|label | L | +|line | I | +|============================= diff --git a/beam_instructions.asciidoc b/beam_instructions.asciidoc new file mode 100755 index 0000000..b010997 --- /dev/null +++ b/beam_instructions.asciidoc @@ -0,0 +1,503 @@ + +[[CH-Instructions]] +== Generic BEAM Instructions (25p) + +Beam has two different instructions sets, an internal instructions +set, called _specific_, and an external instruction set, called +_generic_. + +The generic instruction set is what could be called the official +instruction set, this is the set of instructions used by both the +compiler and the Beam interpreter. If there was an official Erlang +Virtual Machine specification it would specify this +instruction set. If you want to write your own compiler to the Beam, +this is the instruction set you should target. If you want to write +your own EVM this is the instruction set you should handle. + +The external instruction set is quite stable, but it does change +between Erlang versions, especially between major versions. + +This is the instruction set which we will cover in this chapter. + +The other instruction set, the specific, is an optimized instruction +set used by the Beam to implement the external instruction set. To +give you an understanding of how the Beam works we will cover this +instruction set in xref:CH-Internal_instructions[]. The internal +instruction set can change without warning between minor version or +even in patch releases. Basing any tool on the internal instruction +set is risky. + +In this chapter I will go through the general syntax for the +instructions and some instruction groups in detail, a complete list of +instructions with short descriptions can be found in +xref:AP-Instructions[]. + +=== Instruction definitions + +The names and opcodes of the generic instructions are defined +in +lib/compiler/src/genop.tab+. + +The file contains a version number for the Beam instruction format, which +also is wirtten to +.beam+ files. This number has so far never changed +and is still at version 0. If the external format would be changed in a +non backwards compatible way this number would be changed. + +The file genop.tab is used as input by +beam_makeops+ which is a perl script +which generate code from the ops tabs. The generator is used both to generate +Erlang code for the compiler (beam_opcodes.hrl and beam_opcodes.erl) and to +generate C code for the emulator ( TODO: Filenames). + +Any line in the file starting with "#" is a comment and ignored by ++beam_makeops+. The file can contain definitions, which turns into a +binding in the perl script, of the form: + + NAME=EXPR + +Like, e.g.: +---- +BEAM_FORMAT_NUMBER=0 +---- +The Beam format number is the same as the +instructionset+ field in +the external beam format. It is only bumped when a backwards +icnompatible change to the instruction set is made. + +The main content of the file are the opcode definitions of the form: +---- +OPNUM: [-]NAME/ARITY +---- +Where OPNUM and ARITY are integers, NAME is an identifier starting +with a lowercase letter (a-z), and _:_, _-_, and _/_ are litterals. + +For example: +---- +1: label/1 +---- + +The minus sign (-) indicates a depricated function. A depricated +function keeps its opcode in order for the loader to be sort of +backwards compatible (it will recognize depricated instructions and +refuse to load the code). + +In the rest of this Chapter we will go through some BEAM instructions +in detail. For a full list with brief descriptions see: +xref:AP-Instructions[]. + +=== BEAM code listings +As we saw in xref:CH-Compiler[] we can give the option 'S' to the +Erlang compiler to get a +.S+ file with the BEAM code for the module +in a human and machine readable format (actually as Erlang terms). + +Given the file beamexample1.erl: + +++++ +-module(beamexample1). + +-export([id/1]). + +id(I) when is_integer(I) -> I. +++++ + +When compiled with +erlc -S beamexample.erl+ we get the following +beamexmaple.S file: + +++++ +{module, beamexample1}. %% version = 0 + +{exports, [{id,1},{module_info,0},{module_info,1}]}. + +{attributes, []}. + +{labels, 7}. + + +{function, id, 1, 2}. + {label,1}. + {line,[{location,"beamexample1.erl",5}]}. + {func_info,{atom,beamexample1},{atom,id},1}. + {label,2}. + {test,is_integer,{f,1},[{x,0}]}. + return. + + +{function, module_info, 0, 4}. + {label,3}. + {line,[]}. + {func_info,{atom,beamexample1},{atom,module_info},0}. + {label,4}. + {move,{atom,beamexample1},{x,0}}. + {line,[]}. + {call_ext_only,1,{extfunc,erlang,get_module_info,1}}. + + +{function, module_info, 1, 6}. + {label,5}. + {line,[]}. + {func_info,{atom,beamexample1},{atom,module_info},1}. + {label,6}. + {move,{x,0},{x,1}}. + {move,{atom,beamexample1},{x,0}}. + {line,[]}. + {call_ext_only,2,{extfunc,erlang,get_module_info,2}}. +++++ + +In addition to the actual beam code for the integer identity +function we also get some meta instructions. + +The first line +{module, beamexample1}. %% version = 0+ tells +us the module name "beamexample1" and the version number for +the instruction set "0". + +Then we get a list of exported functions "id/1, module_info/0, +module_info/1". As we can see the compiler has added two auto +generated functions to the code. These two functions are just +dispatchers to the generic module info BIFs (erlang:module_info/1 and +erlang:module_info/2) with the name of the module added as the first +argument. + +The line {attributes, []} list all defined compiler attributes, none in +our case. + +Then we get to know that there are less than 7 labels in the module, ++{labels, 7}+, which makes it easy to do code loading in one pass. + +The last type of meta instruction is the +function+ instruction on +the format +{function, Name, Arity, StartLabel}+. As we can see with +the +id+ function the start label is actually the second label in the +code of the function. + +The instruction +{label, N}+ is not really an instruction, it does not +take up any space in memory when loaded. It is just to give a local +name (or number) to a position in the code. Each label potentially +marks the beginning of a basic block since it is a potential +destination of a jump. + +The first two instructions following the first label (+{label,1}+) +are actually error generating code which adds the line number and +module, function and arity information and throws an exception. +That are the instructions +line+ and +func_info+. + +The meat of the function is after +{label,2}+, the instruction ++{test,is_integer,{f,1},[{x,0}]}+. The test instruction tests if its +arguments (in the list at the end, that is variable {x,0} in this +case) fulfills the test, in this case is an integer (is_integer). +If the test succeeds the next instruction (+return+) is executed. +Otherwise the functions fails to label 1 (+{f,1}+), that is, +execution continues at label one where a function clause exception +is thrown. + +The other two functions in the file are auto generated. If we look at +the second function the instruction +{move,{x,0},{x,1}}+ moves the +argument in register x0 to the second argument register x1. Then the +instruction +{move,{atom,beamexample1},{x,0}}+ moves the module name +atom to the first argument register x1. Finally a tail call is made to ++erlang:get_module_info/2+ +(+{call_ext_only,2,{extfunc,erlang,get_module_info,2}}+). As we will +see in the next section there are several different call instructions. + +=== Calls + +As we have seen in xref:CH-Calls[] there are several different types +of calls in Erlang. To distinguish between local and remote calls +in the instruction set, remote calls have +_ext+ in their instruction +names. Local calls just have a label in the code of the module, while +remote calls takes a destination of the form +{extfunc, Module, Function, +Arity}+. + +To distinguish between ordinary (stack building) calls and +tail-recursive calls, the latter have either +_only+ or +_last+ in +their name. The variant with +_last+ will also deallocate as many +stack slot as given by the last argument. + +There is also a +call_fun Arity+ instruction that calls the closure +stored in register {x, Arity}. The arguments are stored in x0 to {x, +Arity-1}. + +For a full listing of all types of call instructions see +xref:AP-Instructions[]. + +=== Stack (and Heap) Management + +The stack and the heap of an Erlang process on Beam share the same memory +area see xref:CH-Processes[] and xref:CH-Memory[] for a full discussion. +The stack grows toward lower addresses and the heap toward higher addresses. +Beam will do a garbage collection if more space than what is available is +needed on either the stack or the heap. + +************************** + +*A leaf function*:: A leaf function is a function which doesn't call + any other function. + +*A non leaf function*:: A non leaf function is a function which may call + another function. + +************************** + + +On entry to a non leaf function the _continuation pointer_ (CP) is saved on +the stack, and on exit it is read back from the stack. This is done by the ++allocate+ and +deallocate+ instructions, which are used for setting up +and tearing down the stack frame for the current instruction. + +A function skeleton for a leaf function looks like this: + +++++ +{function, Name, Arity, StartLabel}. + {label,L1}. + {func_info,{atom,Module},{atom,Name},Arity}. + {label,L2}. + ... + return. +++++ + + +A function skeleton for a non leaf function looks like this: + +++++ +{function, Name, Arity, StartLabel}. + {label,L1}. + {func_info,{atom,Module},{atom,Name},Arity}. + {label,L2}. + {allocate,Need,Live}. + + ... + call ... + ... + + {deallocate,Need}. + return. +++++ + +The instruction +allocate StackNeed Live+ saves the continuation +pointer (CP) and allocate space for +StackNeed+ extra words on the +stack. If a GC is needed during allocation save +Live+ number of X +registers. E.g. if +Live+ is 2 then registers X0 and X1 are saved. + +When allocating on the stack, the stack pointer (E) is decreased. + +.Allocate 1 0 +++++ + Before After + | xxx | | xxx | + E -> | xxx | | xxx | + | | | ??? | caller save slot + ... E -> | CP | + ... ... + HTOP -> | | HTOP -> | | + | xxx | | xxx | +++++ + +For a full listing of all types of allocate and deallocate +instructions see xref:AP-Instructions[]. + + +=== Message Passing + +Sending a message is straight forward in beam code. You just use the ++send+ instruction. Note though that the send instruction does not +take any arguments, it is more like a function call. It assumes that +the arguments (the destination and the message) are in the argument +registers X0 and X1. The message is also copied from X1 to X0. + +Receiving a message is a bit more complicated since it involves both +selective receive with pattern matching and introduces a yield/resume +point within a function body. (There is also a special feature to +minimize message queue scanning using refs, more on that later.) + +==== A Minimal Receive Loop + +A minimal receive loop, which accepts any message and has no timeout +(e.g. +receive _ -> ok end+) looks like this in BEAM code: + +++++ +
+A simple receive loop. + + + L1: loop_rec L2 x0 + remove_message + ... + jump L3 + + L2: wait L1 + + L3: ... + +
+++++ + +The +loop_rec L2 x0+ instruction first checks if there is any message +in the message queue. If there are no messages execution jumps to L2, +where the process will be suspended waiting for a message to arrive. + +If there is a message in the message queue the +loop_rec+ instruction +also moves the message from the _m-buf_ to the process heap. See +xref:CH-Memory[] and xref:CH-Processes[] for details of the m-buf +handling. + +For code like +receive _ -> ok end+, where we accept any messages, +there is no pattern matching needed, we just do a +remove_message+ +which unlinks the next message from the message queue and stores a +pointer in X0. (It also removes any timeout, more on this soon.) + +==== A Selective Receive Loop + +For a selective receive like e.g. +receive [] -> ok end+ we will +loop over the message queue to check if any message in the queue +matches. + +++++ +
+A selective receive loop. + + + L2: loop_rec L4 X0 + test is_nil L3 X0 + remove_message + move {atom,ok} X0 + return + L3: loop_rec_end L2 + L4: wait L2 + + +
+++++ + +In this case we do a pattern match for Nil after the loop_rec +instruction if there was a message in the mailbox. If the message +doesn't match we end up at L3 where the instruction +loop_rec_end+ +advances the save pointer to the next message (+p->msg.save = +&(*p->msg.save)->next+) and jumps back to L2. + +If there are no more messages in the message queue the process is +suspended by the +wait+ instruction at L4 with the save pointer pointing +to the end of the message queue. When the processes is rescheduled +it will only look at new messages in the message queue (after the save +point). + +==== A Receive Loop With a Timeout + +If we add a timeout to our selective receive the wait instruction is +replaced by a wait_timeout instruction followed by a timeout +instruction and the code following the timeout. + + +++++ +
+A receive loop with a timeout. + + + L6: loop_rec L8 X0 + test is_nil L7 X0 + remove_message + move {atom,ok} X0 + return + L7: loop_rec_end L6 + L8: wait_timeout L6 {integer,1000} + timeout + move {atom,done} X0 + return + + +
+++++ + +The +wait_timeout+ instructions sets up a timeout timer with the given +time (1000 ms in our example) and it also saves the address of the +next instruction (the +timeout+) in +p->def_arg_reg[0]+ and then +when the timer is set, +p->i+ is set to point to def_arg_reg. + +This means that if no matching message arrives while the process is +suspended a timeout will be triggered after 1 second and execution for +the process will continue at the timeout instruction. + +Note that if a message that doesn't match arrives in the mailbox, the +process is scheduled for execution and will run the pattern matching +code in the receive loop, but the timeout will not be canceled. It is +the +remove_message+ code which also removes any timeout timer. + +The +timeout+ instruction resets the save point of the mailbox to the +first element in the queue, and clears the timeout flag (F_TIMO) from +the PCB. + +==== The Synchronous Call Trick (aka The Ref Trick) + +We have now come to the last version of our receive loop, where we +use the ref trick alluded to earlier to avoid a long message box scan. + +A common pattern in Erlang code is to implement a type of "remote +call" with send and a receive between two processes. This is for +example used by gen_server. This code is often hidden behind a library +of ordinary function calls. E.g., you call the function ++counter:increment(Counter)+ and behind the scene this turns into +something like +Counter ! {self(), inc}, receive {Counter, Count} -> +Count end+. + +This is usually a nice abstraction to encapsulate state in a +process. There is a slight problem though when the mailbox of the +calling process has many messages in it. In this case the receive will +have to check each message in the mailbox to find out that no message +except the last matches the return message. + +This can quite often happen if you have a server that receives many +messages and for each message does a number of such remote calls, if +there is no back throttle in place the servers message queue will +fill up. + +To remedy this there is a hack in ERTS to recognize this pattern and +avoid scanning the whole message queue for the return message. + +The compiler recognizes code that uses a newly created reference (ref) +in a receive (see xref:ref_trick_code[]), and emits code to avoid the +long inbox scan since the new ref can not already be in the inbox. + +++++ +
+The Ref Trick Pattern. + + + Ref = make_ref(), + Counter ! {self(), inc, Ref}, + receive + {Ref, Count} -> Count + end. + +
+++++ + +This gives us the following skeleton for a complete receive, see +xref:ref_receive[]. + +++++ +
+A receive with the ref_trick. + + ... + recv_mark L11 + call_ext 0 {extfunc,erlang,make_ref,0} + ... + + recv_set L11 +L11: loop_rec L13 x0 + test is_eq_exact L12 [X0, Y0] + remove_message + ... + +L12: loop_rec_end L11 +L13: wait_timeout L11 {integer,1000} + timeout. + ... + + +
+++++ + +The +recv_mark+ instruction saves the current position (the end ++msg.last+) in +msg.saved_last+ and the address of the label +in +msg.mark+ + +The +recv_set+ instruction checks that +msg.mark+ points to the next +instruction and in that case moves the save point (+msg.save+) to the +last message received before the creation of the ref +(+msg.saved_last+). If the mark is invalid ( i.e. not equal to ++msg.save+) the instruction does nothing. diff --git a/beam_internal_instructions.asciidoc b/beam_internal_instructions.asciidoc new file mode 100755 index 0000000..0c7a940 --- /dev/null +++ b/beam_internal_instructions.asciidoc @@ -0,0 +1,4 @@ + +[[CH-Internal-instructions]] +== BEAM Internal Instructions + diff --git a/book.asciidoc b/book.asciidoc index 6748e02..f372d4c 100644 --- a/book.asciidoc +++ b/book.asciidoc @@ -10,23 +10,23 @@ include::compiler.asciidoc[] include::processes.asciidoc[] -// include::type_system.asciidoc[] +include::type_system.asciidoc[] // include::beam.asciidoc[] // include::beam_modules.asciidoc[] -// include::beam_instructions.asciidoc[] +include::beam_instructions.asciidoc[] -// include::calls.asciidoc[] +include::calls.asciidoc[] // include::beam_loader.asciidoc[] -// include::beam_internal_instructions.asciidoc[] +include::beam_internal_instructions.asciidoc[] // include::scheduling.asciidoc[] -// include::memory.asciidoc[] +include::memory.asciidoc[] // include::data_structures.asciidoc[] @@ -46,12 +46,12 @@ include::processes.asciidoc[] // include::tweak.asciidoc[] -// // Appendix +[appendix] // include::index.asciidoc[] -// include::ap-beam_instructions.asciidoc[] +include::ap-beam_instructions.asciidoc[] // include::ap-code_listings.asciidoc[] diff --git a/calls.asciidoc b/calls.asciidoc new file mode 100755 index 0000000..b065b70 --- /dev/null +++ b/calls.asciidoc @@ -0,0 +1,149 @@ + +[[CH-Calls]] +== Different Types of Calls, Linking and Hot Code Loading (5p) + +**** +Local calls, remote calls, closure calls, tuple calls, p-mod +calls. The code server. Linking. Hot code loading, purging. (Overlap +with Chapter 4, have to see what goes where.) Higher order functions, +Implementation of higher order functions. Higher order functions and +hot code loading. Higher order functions in a distributed system. +**** + +=== Hot Code Loading + +In Erlang there is a semantic difference between a local function call +and a remote function call. A remote call, that is a call to a +function in a named module, is guaranteed to go to the latest loaded +version of that module. A local call, a unqualified call to a function +within the same module, is guaranteed to go to the same version +of the code as the caller. + +A call to a local function can be turned into a remote call by +specifying the module name at the call site. This is usually +done with the ?MODULE macro as in +?MODULE:foo()+. +A remote call to a non local module can not be turned into +a local call, i.e. there is no way to guarantee the version +of the callee in the caller. + +This is an important feature of Erlang which makes _hot code loading_ +or _hot upgrades_ possible. Just make sure you have a remote +call somewhere in your server loop and you can then load new code +into the system while it is running; when execution reaches the +remote call it will switch to executing the new code. + +A common way of writing server loops is to have a local call +for the main loop and a code upgrade handler which does +a remote call and possibly a state upgrade: + +[source,erlang] +------------------------------------------ +loop(State) -> + receive + upgrade -> + NewState = ?MODULE:code_upgrade(State), + ?MODULE:loop(NewState); + Msg -> + NewState = handle_msg(Msg, State), + loop(NewState) + end. + +------------------------------------------ + +With this construct, which is basically what gen_server uses, +the programmer has control over when and how a code upgrade is done. + +The hot code upgrade is one of the most important features of Erlang +which makes it possible to write servers that operates 24/7 year out +and year in. It is also one of the main reasons why Erlang is +dynamically typed. It is very hard in a statically typed language to +give type for the code_upgrade function. (It is also hard to give the +type of the loop function) These types will change in the future as +the type of State changes to handle new features. + +For a language implementer concerned with performance, the hot code +loading functionality is a burden though. Since each call to or from a +remote module can change to new code in the future it is very hard to +do whole program optimization across module boundaries. (Hard but not +impossible, there are solutions but so far I have not seen one fully +implemented.) + +=== Code Loading + +++++ + +++++ + + +In the Erlang Runtime System the code loading is handled by the +code server. The code server will call the lover level bifs in the ++erlang+ module for the actual loading. But the code server also +determines the purging policy. + +The runtime system can keep two versions of each module, a _current_ +version and an _old_ version. All fully qualified (remote) calls goes +to the current version. Local calls in the old version and return +addresses on the stack can still go to the old version. + +If a third version of a module is loaded and there still are processes +running (have pointers on the stack to) old code the code server +will kill does processes and purge the old code. Then the current +version will become old and the third version will be loaded as the +current version. + + +//[[CH-Beam_loader]] +// === The BEAM Loader + +// Translation to internal format. +// Optimizations. +// Rewrites +// Peephole optimisztions +// pack engine? +// +// ops.tab format/syntax +// Catches +// Linking and Exports + + +=== Transforming from Generic to Specific instructions + +The BEAM loader does not just take the external beam format and writes +it to memory. It also does a number of transformations on the code +and translates from the external (generic) format to the internal +(specific) format. + +The code for the loader can be found in +beam_load.c+ (in ++erts/emulator/+) but most of the logic for the translations are in +the file +ops.tab+ (in the same directory). + +The first step of the loader is to parse beam file, basically the same +work as we did in Erlang in xref:CH-beam_modules[] but written in C. + +Then the rules in ops.tab are applied to instructions in the code +chunk to translate the generic instruction to one or more specific +instructions. + +The translation table works through pattern matching. Each line in the +file defines a pattern of one or more generic instructions with +arguments and optionally an arrow followed by one or more instructions +to translate to. + +The transformations in ops tab tries to handle patterns of +instructions generated by the compiler and peephole optimize them to +fewer specific instructions. The ops tab transformations tries to +generate jump tables for patterns of selects. + +The file ops.tab is not parsed at runtime, instead a pattern matching +program is generated from ops.tab and stored in an array in a +generated C file. The perl script +beam_makeops+ (in ++erts/emulator/utils+) generates a target specific set of opcodes and +translation programs in the files +beam_opcodes.h+ and ++beam_opcodes.c+ (these files end up in the given target directory +e.g. +erts/emulator/x86_64-unknown-linux-gnu/opt/smp/+). + +The same program (beam_makeops) also generates the Erlang code for the +compiler back end +beam_opcodes.erl+. + diff --git a/code/book/src/generate_op_doc.erl b/code/book/src/generate_op_doc.erl new file mode 100755 index 0000000..7ff8ce0 --- /dev/null +++ b/code/book/src/generate_op_doc.erl @@ -0,0 +1,120 @@ +-module(generate_op_doc). + +-export([docbook/2, from_shell/1]). + +-record(op, {name="", arity="", opcode, doc="", spec="", deprecated=false}). + +from_shell([In, Out]) -> + docbook(atom_to_list(In), atom_to_list(Out)), + halt(). + +docbook(InFile, OutFile) -> + {ok, File} = file:open(InFile, [read]), + Ops = parse(File), + Doc = docbook_format(Ops), + file:write_file(OutFile, Doc). + + + +docbook_format(Ops) -> + docbook_format_line(lists:reverse(lists:keysort(2,Ops))) + ++ + "|=================================================\n". + +docbook_format_line([#op{name=Name, + arity=Arity, + opcode=Opcode, + doc=Doc, + spec=Spec, + deprecated=Deprecated}| + Prev]) -> + docbook_format_line(Prev) ++ + "|" ++ format_name(Name, Deprecated) ++ + "|" ++ strip(Arity) ++ + "|" ++ format_opcode(Opcode, Deprecated) ++ + "|" ++ format_spec(Spec, Deprecated) ++ + "|" ++ strip(Doc) ++ "\n"; +docbook_format_line([]) -> + "=== Generic Instructions\n" ++ + "[options=" ++ [$"] ++ "header" ++ [$"] ++ "]\n" + "|=================================================\n" ++ + "| Name | Arity | Op Code | Spec | Documentation\n". + + +format_name(Name, Deprecated) -> + if + Deprecated -> "[line-through]#" ++ strip(Name) ++ "#"; + true -> strip(Name) + end. + +format_spec([Name|Args], false) -> + "*"++strip(Name)++"*" ++ " " + ++ string:join([format_arg(A) || A<-Args], ", "); +format_spec(_, true) -> "*DEPRECATED*"; +format_spec([], _) -> "". + +format_arg(A) -> "_"++strip(A)++"_". + +format_opcode(undefined, _Deprecated) -> + ""; +format_opcode(Opcode, Deprecated) -> + if + Deprecated -> "("; + true -> "" + end ++ + strip(Opcode) ++ + if + Deprecated -> ")"; + true -> "" + end. + + + +strip(S) -> + [ esacpe(Char) + || Char <- string:strip(S, right, 10)]. + +esacpe(Char) -> + case Char of + $| -> "\|"; + $\n -> " "; + _ -> Char + end. + +parse(File) -> + parse(File, #op{}, []). + +parse(File, Op, Ops) -> + case file:read_line(File) of + {ok, Line} -> + {NewOp, NewOps} = parse_line(Line, Op, Ops), + parse(File, NewOp, NewOps); + eof -> case Op#op.name of + "" -> Ops; + _ -> [Op|Ops] + end + end. + +parse_line("##" ++ Rest, Op, Ops) -> + {parse_doc(Rest, Op), Ops}; +parse_line("#" ++ _, Op, Ops) -> {Op, Ops}; +parse_line([N|_]=Line, Op, Ops) when N >= $0, N =< $9 -> + [OpNo, NA] = string:tokens(Line, ":"), + [Name, Arity] = string:tokens(string:strip(NA, left, $ ), "/"), + NewOp = + case Name of + "-" ++ OpName -> + Op#op{name=OpName, deprecated=true}; + _ -> + Op#op{name=Name} + end, + {#op{}, [NewOp#op{opcode=OpNo, arity=Arity} | Ops]}; +parse_line(_, Op, Ops) -> + {Op, Ops}. + +parse_doc(Line, Op) -> + case string:tokens(Line, " ") of + ["@spec" | Rest] -> Op#op{spec=Rest}; + ["@doc" | Rest] -> Op#op{doc=string:join(Rest, " ")}; + _ -> Op#op{doc=Op#op.doc++Line} + end. diff --git a/code/compiler_chapter/src/json_tokens.xrl b/code/compiler_chapter/src/json_tokens.xrl new file mode 100644 index 0000000..31569ba --- /dev/null +++ b/code/compiler_chapter/src/json_tokens.xrl @@ -0,0 +1,68 @@ +Definitions. + +Digit = [0-9] +Digit1to9 = [1-9] +HexDigit = [0-9a-f] +UnescapedChar = [^\"\\] +EscapedChar = (\\\\)|(\\\")|(\\b)|(\\f)|(\\n)|(\\r)|(\\t)|(\\/) +Unicode = (\\u{HexDigit}{HexDigit}{HexDigit}{HexDigit}) +Quote = [\"] +Delim = [\[\]:,{}] +Space = [\n\s\t\r] + +Rules. + +{Quote}{Quote} : {token, {string, TokenLine, ""}}. +{Quote}({EscapedChar}|({UnescapedChar})|({Unicode}))+{Quote} : + {token, {string, TokenLine, drop_quotes(TokenChars)}}. + +null : {token, {null, TokenLine}}. +true : {token, {true, TokenLine}}. +false : {token, {false, TokenLine}}. + +{Delim} : {token, {list_to_atom(TokenChars), TokenLine}}. + +{Space} : skip_token. + +-?{Digit1to9}+{Digit}*\.{Digit}+((E|e)(\+|\-)?{Digit}+)? : + {token, {number, TokenLine, list_to_float(TokenChars)}}. +-?{Digit1to9}+{Digit}* : + {token, {number, TokenLine, list_to_integer(TokenChars)+0.0}}. + +Erlang code. +-export([t/0]). + +drop_quotes([$" | QuotedString]) -> literal(lists:droplast(QuotedString)). +literal([$\\,$" | Rest]) -> + [$"|literal(Rest)]; +literal([$\\,$\\ | Rest]) -> + [$\\|literal(Rest)]; +literal([$\\,$/ | Rest]) -> + [$/|literal(Rest)]; +literal([$\\,$b | Rest]) -> + [$\b|literal(Rest)]; +literal([$\\,$f | Rest]) -> + [$\f|literal(Rest)]; +literal([$\\,$n | Rest]) -> + [$\n|literal(Rest)]; +literal([$\\,$r | Rest]) -> + [$\r|literal(Rest)]; +literal([$\\,$t | Rest]) -> + [$\t|literal(Rest)]; +literal([$\\,$u,D0,D1,D2,D3|Rest]) -> + Char = list_to_integer([D0,D1,D2,D3],16), + [Char|literal(Rest)]; +literal([C|Rest]) -> + [C|literal(Rest)]; +literal([]) ->[]. + +t() -> + {ok, + [{'{',1}, + {string,2,"no"}, + {':',2}, + {number,2,1.0}, + {'}',3} + ], + 4}. + diff --git a/code/compiler_chapter/src/world.E b/code/compiler_chapter/src/world.E new file mode 100644 index 0000000..def7d8f --- /dev/null +++ b/code/compiler_chapter/src/world.E @@ -0,0 +1,17 @@ +-vsn("\002"). + +-file("world.erl", 1). + +-file("world.hrl", 1). + +-file("world.erl", 5). + +hello() -> + "hello world". + +module_info() -> + erlang:get_module_info(world). + +module_info(X) -> + erlang:get_module_info(world, X). + diff --git a/code/compiler_chapter/src/world.P b/code/compiler_chapter/src/world.P new file mode 100644 index 0000000..31904ad --- /dev/null +++ b/code/compiler_chapter/src/world.P @@ -0,0 +1,15 @@ +-file("world.erl", 1). + +-module(world). + +-export([hello/0]). + +-file("world.hrl", 1). + +-file("world.erl", 4). + +hello() -> + "hello world". + + + diff --git a/code/compiler_chapter/src/world.S b/code/compiler_chapter/src/world.S new file mode 100644 index 0000000..7a67d7e --- /dev/null +++ b/code/compiler_chapter/src/world.S @@ -0,0 +1,37 @@ +{module, world}. %% version = 0 + +{exports, [{hello,0},{module_info,0},{module_info,1}]}. + +{attributes, []}. + +{labels, 7}. + + +{function, hello, 0, 2}. + {label,1}. + {line,[{location,"world.erl",6}]}. + {func_info,{atom,world},{atom,hello},0}. + {label,2}. + {move,{literal,"hello world"},{x,0}}. + return. + + +{function, module_info, 0, 4}. + {label,3}. + {line,[]}. + {func_info,{atom,world},{atom,module_info},0}. + {label,4}. + {move,{atom,world},{x,0}}. + {line,[]}. + {call_ext_only,1,{extfunc,erlang,get_module_info,1}}. + + +{function, module_info, 1, 6}. + {label,5}. + {line,[]}. + {func_info,{atom,world},{atom,module_info},1}. + {label,6}. + {move,{x,0},{x,1}}. + {move,{atom,world},{x,0}}. + {line,[]}. + {call_ext_only,2,{extfunc,erlang,get_module_info,2}}. diff --git a/code/compiler_chapter/src/world.erl b/code/compiler_chapter/src/world.erl new file mode 100644 index 0000000..e45b464 --- /dev/null +++ b/code/compiler_chapter/src/world.erl @@ -0,0 +1,7 @@ +-module(world). +-export([hello/0]). + +-include("world.hrl"). + +hello() -> ?GREETING. + diff --git a/code/compiler_chapter/src/world.hrl b/code/compiler_chapter/src/world.hrl new file mode 100644 index 0000000..ad003c7 --- /dev/null +++ b/code/compiler_chapter/src/world.hrl @@ -0,0 +1 @@ +-define(GREETING, "hello world"). diff --git a/code/memory_chapter/src/gc_example.erl b/code/memory_chapter/src/gc_example.erl new file mode 100644 index 0000000..29d7202 --- /dev/null +++ b/code/memory_chapter/src/gc_example.erl @@ -0,0 +1,19 @@ +-module(gc_example). +-export([example/0]). + +example() -> + T = gen_data(), + S = element(1, T), + erlang:garbage_collect(), + S. + +gen_data() -> + S = gen_string($H, $e, $l, $l, $o), + T = gen_tuple([S,S],S), + T. + +gen_string(A,B,C,D,E) -> + [A,B,C,D,E]. + +gen_tuple(A,B) -> + {A,B}. diff --git a/compiler.asciidoc b/compiler.asciidoc index 5e147c2..b7a92d5 100755 --- a/compiler.asciidoc +++ b/compiler.asciidoc @@ -1,773 +1,764 @@ [[ch.compiler]] == The Compiler -// This book will not cover the programming language Erlang, but since -// the goal of the ERTS is to run Erlang code you will need to know how to -// compile Erlang code. In this chapter I will cover the compiler -// options needed to generate readable beam code and how to add -// debug information to the generated beam file. - -// For those readers interested in compiling their own favorite language -// to ERTS this chapter will also contain detailed information about the -// different intermediate formats in the compiler and how to plug your -// compiler into the beam compiler backend. I will also present parse -// transforms and give examples of how to use them to tweak the Erlang -// language. - -// If your main interest in reading this book is to understand -// the Erlang VM and how to debug and tweak it, you can safely -// skip this chapter. - - -// === Compiling Erlang - -// Erlang is compiled from source code modules in +.erl+ files -// to fat binary +.beam+ files. - -// The compiler can be run from the OS shell with the +erlc+ command: -// [source,bash] -// ---- -// > erlc foo.erl -// ---- - -// Alternatively the compiler can be invoked from the Erlang shell with -// the default shell command +c+ or by calling +compile:file/{1,2}+ - -// [source,erlang] -// ---- -// 1> c(foo). -// ---- -// or - -// [source,erlang] -// ---- -// 1> compile:file(foo). -// ---- - -// The optional second argument to +compile:file+ is a list of compiler -// options. A full list of the options can be found in the documentation -// of the compile module: see link:http://www.erlang.org/doc/man/compile.html[]. - -// Normally the compiler will compile Erlang source code from a +.erl+ -// file and write the resulting binary beam code to a +.beam+ file. You -// can also get the resulting binary back as an Erlang term -// by giving the option +binary+ to the compiler. This -// option has then been overloaded to mean return any intermediate format -// as a term instead of writing to a file. If you for example want the -// compiler to return Core Erlang code you can give the options +[core, -// binary]+. - - -// The compiler is made up of a number of passes as illustrated in -// xref:fig_compiler_passes[Compiler Passes]. - - -// // Compiler Passes. [] = Compiler options, () = files, {} = erlang terms, boxes = passes - -// [[fig_compiler_passes]] -// ++++ -// 
       (.erl)
-//            |
-//            v
-//    +---------------+
-//    |    Scanner    |
-//    | (Part of epp) |
-//    +---------------+
-//            |
-//            v
-//    +---------------+
-//    | Pre-processor |
-//    |      epp      |
-//    +---------------+
-//            |
-//            v
-//    +---------------+    +---------------+    
-//    |     Parse     | -> | user defined  |
-//    |   Transform   | <- | transformation|
-//    +---------------+    +---------------+
-//            |
-//            +---------> (.Pbeam) [makedep]
-//            +---------> {dep} [makedep, binary]
-//            |
-//            +---------> (.pp) [dpp]
-//            +---------> {AST} [dpp, binary]
-//            |
-//            v
-//    +---------------+
-//    |    Linter     |
-//    |               |
-//    +---------------+
-//            |
-//            +---------> (.P) ['P']
-//            +---------> {AST} ['P',binary]
-//            |
-//            v
-//    +---------------+
-//    |    Save AST   |
-//    |               |
-//    +---------------+
-//            |
-//            v
-//    +---------------+
-//    |     Expand    |
-//    |               |
-//    +---------------+
-//            |
-//            +---------> (.E) ['E']
-//            +---------> {.E} ['E', binary]
-//            |
-//            v
-//    +---------------+
-//    |     Core      |
-//    |    Erlang     |
-//    +---------------+
-//            |
-//            +---------> (.core) [dcore|to_core0]
-//            +---------> {core} [to_core0,binary]
-//            |
-//            v
-//    +---------------+
-//    |      Core     |+
-//    |     Passes    ||+
-//    +---------------+||
-//     +---------------+|
-//      +---------------+
-//            |
-//            +---------> (.core) [to_core]
-//            +---------> {core} [to_core,binary]
-//            |
-//            v
-//    +---------------+
-//    |    Kernel     |
-//    |    Erlang     |
-//    +---------------+
-//            |
-//            v
-//    +---------------+
-//    |    Kernel     |+
-//    |    Passes     ||+
-//    +---------------+||
-//     +---------------+|
-//      +---------------+
-//            |
-//            v
-//    +---------------+
-//    |   BEAM Code   |
-//    |               |
-//    +---------------+
-//            |
-//            v
-//    +---------------+
-//    |      ASM      |+
-//    |     Passes    ||+
-//    +---------------+||
-//     +---------------+|
-//      +---------------+
-//            |
-//            +---------> (.S) ['S']
-//            +---------> {.S} ['S', binary]
-//            |
-//            v
-//    +---------------+
-//    |  Native Code  |
-//    |               |
-//    +---------------+
-//            |
-//            v
-//         (.beam)
-// ++++ -// // - -// If you want to see a complete and up to date list of compiler passes -// you can run the function +compile:options/0+ in an Erlang shell. -// The definitive source for information about the compiler is of course -// the source: -// link:https://github.com/erlang/otp/blob/maint/lib/compiler/src/compile.erl[compile.erl] - - - -// === Generating Intermediate Output - -// Looking at the code produced by the compiler is a great help in trying -// to understand how the virtual machine works. Fortunately, the compiler -// can show os the intermediate code after each compiler pass and the -// final beam code. - -// Let us try out our newfound knowledge to look at the generated code. - - -// [source,erlang] -// ---- -// 1> compile:options(). -// dpp - Generate .pp file -// 'P' - Generate .P source listing file -// ---- -// ... -// ---- -// 'E' - Generate .E source listing file -// ---- -// ... -// ---- -// 'S' - Generate .S file -// ---- - -// Let us try with a small example program "world.erl": -// [source,erlang] -// ---- -// include::world.erl[] -// ---- - -// And the include file "world.hrl" -// [source,erlang] -// ---- -// include::world.hrl[] -// ---- - -// If you now compile this with the 'P' option to get the parsed file you -// get a file "world.P": - -// [source,erlang] -// ---- -// 2> c(world, ['P']). -// ** Warning: No object file created - nothing loaded ** -// ok -// ---- - -// In the resulting +.P+ file you can see the a pretty printet version of -// the code after the preprocessor (and parse transformation) has been -// applied: - -// [source,erlang] -// ---- -// include::world.P[] -// ---- - -// To see how the code looks after all source code transformations are -// done, you can compile the code with the +'E'+-flag. - -// [source,erlang] -// ---- -// 3> c(world, ['E']). -// ** Warning: No object file created - nothing loaded ** -// ok -// ---- - -// This gives us an +.E+ file, in this case all compiler directives have -// been removed and the build in functions +module_info/{1,2}+ have been -// added to the source: - -// [source,erlang] -// ---- -// include::world.E[] -// ---- - -// We will make use of the 'P' and 'E' options when we look at parse -// transforms in xref:SEC-parse_transform[], but first we will take a -// look at an "assembler" view of generated BEAM code. Bu giving the -// option +'S'+ to the compiler you get a +.S+ file with Erlang terms -// for each BEAM instruction in the code. - -// [source,erlang] -// ---- -// 3> c(world, ['S']). -// ** Warning: No object file created - nothing loaded ** -// ok -// ---- - -// The file +world.S+ should look like this: - -// [source,erlang] -// ---- -// include::world.S[] -// ---- - -// Since this is a file with dot ("_._") separated Erlang terms, you can -// read the file back into the Erlang shell with: -// ---- -// {ok, BEAM_Code} = file:consult("world.S"). -// ---- - -// The assembler code mostly follows the layout of the original source -// code. - -// The first instruction defines the module name of the code. The version -// mentioned in the comment (+%% version = 0+) is the version of the beam -// opcode format (as given by +beam_opcodes:format_number/0+). - -// Then comes a list of exports and any compiler attributes (none in this -// example) much like in any Erlang source module. - -// The first real beam-like instruction is +{labels, 7}+ which tells the -// VM the number of lables in the code to make it possible to allocate -// room for all lables in one pass over the code. - -// After that there is the actual code for each function. The first -// instruction gives us the function name, the arity and the entry point -// as a label number. - -// You can use the +'S'+ option with great effect to help you understand -// how the BEAM works, and we will use it like that in later chapters. It -// is also invaluable if you develop your own language that you compile -// to the BEAM through Core Erlang, to see the generated code. - -// === Compiler Passes - -// In the following sections we will go through most of the compiler -// passes shown in xref:fig_compiler_passes[]. For a language designer -// targeting the BEAM this is interesting since it will show you what you -// can accomplish with the different approaches: macros, parse -// transforms, core erlang, and BEAM code, and how they depend on each -// other. - -// When tuning Erlang code, it is good to know what optimizations are -// applied when, and how you can look at generated code before and -// after optimizations. - - -// ==== Compiler Pass: The Erlang Preprocessor (epp) - -// The compilation starts with a combined tokenizer (or scanner) and -// preprocessor. That is, the preprosessor drives the tokenizer. -// This means that macros are expanded as tokens, so -// it is not a pure string replacement (as for example m4 or cpp). -// You can not use Erlang macros to define your own syntax, a macro -// will expand as a separate token from its surrounding characters. -// You can not concatenate a macro and a character to a token: - -// ---- -// -define(plus,+). -// t(A,B) -> A?plus+B. -// ---- -// This will expand to -// ---- -// t(A,B) -> A + + B. -// ---- -// and not -// ---- -// t(A,B) -> A ++ B. -// ---- - -// On the other hand since macro expansion is done on the token -// level, you do not need to have a valid Erlang term in the -// right hand side of the macro, as long as you use it in a way -// that gives you a valid term. E.g.: - -// ---- -// -define(p,o, o]). -// t() -> [f,?p. -// ---- - -// I do not know any real use for this other than to win the -// obfuscated Erlang code contest. The main point to remember is that -// you can not really use the Erlang preprocessor to define a language -// with a syntax that differs from Erlang. Fortunately there are -// other ways to do this, as you shall see later. - - - - -// [[SEC-parse_transform]] -// ==== Compiler Pass: Parse Transformations - -// The easiest way to tweak the Erlang language is through Parse -// Transformations (or parse transforms). Parse Transformations comes -// with all sorts of warnings, like this note in the OTP documentation: - -// ---- -// Programmers are strongly advised not to engage in parse -// transformations and no support is offered for problems encountered. -// ---- - -// When you use a parse transform you are basically writing an extra pass -// in the compiler and that can if you are not careful lead to very -// unexpected results. But to use a parse transform you have to declare -// the usage in the module using it, and it will be local to that module, -// so as far as compiler tweaks goes this one is quite safe. - -// The biggest problem with parse transforms as I see it is that you -// are inventing your own syntax, and it will make it more difficult -// for anyone else reading your code. At least until your parse transform -// has become as popular and widely used as e.g. QLC. - -// OK, so you know you shouldn't use it, but if you have to, here is what -// you need to know. A parse transforms is a function that works on the -// abstract syntax tree (AST) (see -// link:http://www.erlang.org/doc/apps/erts/absform.html[] ). The compiler -// does preprocessing, tokenization and parsing and then it will call the -// parse transform function with the AST and expects to get back a -// new AST. - -// This means that you can't change the Erlang syntax fundamentally, but -// you can change the semantics. Lets say for example that you for some -// reason would like to write json code directly in your Erlang code, -// then you are in luck since the tokens of json and of Erlang are -// basically the same. Also, since the Erlang compiler does most of -// its sanity checks in the linter pass which follows the parse transform -// pass, you can allow an AST which does not represent valid Erlang. - -// To write a parse transform you need to write an Erlang module (lets -// call it _p_) which exports the function +parse_transform/2+. This -// function is called by the compiler during the parse transform pass if -// the module being compiled (lets call it _m_) contains the compiler -// option +{parse_transform, p}+. The arguments to the funciton is the -// AST of the module m and the compiler options given to the call to the -// compiler. - -// [NOTE] -// ==== -// Note that you will not get any compiler options given in the file, this -// is a bit of a nuisance since you can't give options to the parse transform -// from the code. - -// The compiler does not expand compiler options until the _expand_ pass -// which occures after the parse transform pass. -// ==== - -// The documenation of the abstract format is somewhat dense and it is -// quite hard to get a grip on the abstract format by reading the -// documentation. I encourage you to use the _syntax_tools_ and -// especially +erl_syntax_lib+ for any serious work on the AST. - -// Here we will develop a a simple parse transform just to get an -// understanding of the AST. Therefore we will work directly on the AST -// and use the old reliable +io:format+ approach instead of syntax_tools. - -// First we create an example of what we would like to be able to compile -// json_test.erl: - -// [source,erlang] -// ---- -// -module(json_test). -// -compile({parse_transform, json_parser}). -// -export([test/1]). - -// test(V) -> -// <<{{ -// "name" : "Jack (\"Bee\") Nimble", -// "format": { -// "type" : "rect", -// "widths" : [1920,1600], -// "height" : (-1080), -// "interlace" : false, -// "frame rate": V -// } -// }}>>. -// ---- - -// Then we create a minimal parse transform module +json_parser.erl+: - -// [source,erlang] -// ---- -// -module(json_parser). -// -export([parse_transform/2]). - -// parse_transform(AST, _Options) -> -// io:format("~p~n", [AST]), -// AST. -// ---- - -// This identity parse transform returns an unchanged AST but it also prints -// it out so that you can see what an AST looks like. - -// ---- -// > c(json_parser). -// {ok,json_parser} -// 2> c(json_test). -// [{attribute,1,file,{"./json_test.erl",1}}, -// {attribute,1,module,json_test}, -// {attribute,3,export,[{test,1}]}, -// {function,5,test,1, -// [{clause,5, -// [{var,5,'V'}], -// [], -// [{bin,6, -// [{bin_element,6, -// {tuple,6, -// [{tuple,6, -// [{remote,7,{string,7,"name"},{string,7,"Jack (\"Bee\") Nimble"}}, -// {remote,8, -// {string,8,"format"}, -// {tuple,8, -// [{remote,9,{string,9,"type"},{string,9,"rect"}}, -// {remote,10, -// {string,10,"widths"}, -// {cons,10, -// {integer,10,1920}, -// {cons,10,{integer,10,1600},{nil,10}}}}, -// {remote,11,{string,11,"height"},{op,11,'-',{integer,11,1080}}}, -// {remote,12,{string,12,"interlace"},{atom,12,false}}, -// {remote,13,{string,13,"frame rate"},{var,13,'V'}}]}}]}]}, -// default,default}]}]}]}, -// {eof,16}] -// ./json_test.erl:7: illegal expression -// ./json_test.erl:8: illegal expression -// ./json_test.erl:5: Warning: variable 'V' is unused -// error -// ---- - -// The compilation of +json_test+ fails since the module contains invalid -// Erlang syntax, but you get to see what the AST looks like. Now we can -// just write some functions to traverse the AST and rewrite the json -// code into Erlang code.footnote:[The translation here is done in -// accordance with EEP 18 (Erlang Enhancement Proposal 18: "JSON bifs") -// link:http://www.erlang.org/eeps/eep-0018.html] - -// [source,erlang] -// ---- -// -module(json_parser). -// -export([parse_transform/2]). - -// parse_transform(AST, _Options) -> -// json(AST, []). - -// -define(FUNCTION(Clauses), {function, Label, Name, Arity, Clauses}). - -// %% We are only interested in code inside functions. -// json([?FUNCTION(Clauses) | Elements], Res) -> -// json(Elements, [?FUNCTION(json_clauses(Clauses)) | Res]); -// json([Other|Elements], Res) -> json(Elements, [Other | Res]); -// json([], Res) -> lists:reverse(Res). - -// %% We are interested in the code in the body of a function. -// json_clauses([{clause, CLine, A1, A2, Code} | Clauses]) -> -// [{clause, CLine, A1, A2, json_code(Code)} | json_clauses(Clauses)]; -// json_clauses([]) -> []. - - -// -define(JSON(Json), {bin, _, [{bin_element -// , _ -// , {tuple, _, [Json]} -// , _ -// , _}]}). - -// %% We look for: <<"json">> = Json-Term -// json_code([]) -> []; -// json_code([?JSON(Json)|MoreCode]) -> [parse_json(Json) | json_code(MoreCode)]; -// json_code(Code) -> Code. - -// %% Json Object -> [{}] | [{Lable, Term}] -// parse_json({tuple,Line,[]}) -> {cons, Line, {tuple, Line, []}}; -// parse_json({tuple,Line,Fields}) -> parse_json_fields(Fields,Line); -// %% Json Array -> List -// parse_json({cons, Line, Head, Tail}) -> {cons, Line, parse_json(Head), -// parse_json(Tail)}; -// parse_json({nil, Line}) -> {nil, Line}; -// %% Json String -> <> -// parse_json({string, Line, String}) -> str_to_bin(String, Line); -// %% Json Integer -> Intger -// parse_json({integer, Line, Integer}) -> {integer, Line, Integer}; -// %% Json Float -> Float -// parse_json({float, Line, Float}) -> {float, Line, Float}; -// %% Json Constant -> true | false | null -// parse_json({atom, Line, true}) -> {atom, Line, true}; -// parse_json({atom, Line, false}) -> {atom, Line, false}; -// parse_json({atom, Line, null}) -> {atom, Line, null}; - -// %% Variables, should contain Erlang encoded Json -// parse_json({var, Line, Var}) -> {var, Line, Var}; -// %% Json Negative Integer or Float -// parse_json({op, Line, '-', {Type, _, N}}) when Type =:= integer -// ; Type =:= float -> -// {Type, Line, -N}. -// %% parse_json(Code) -> io:format("Code: ~p~n",[Code]), Code. - -// -define(FIELD(Lable, Code), {remote, L, {string, _, Label}, Code}). - -// parse_json_fields([], L) -> {nil, L}; -// %% Label : Json-Term --> [{<