From 97e4a0f99e708563397dfa5adf0ec662b3c71485 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 7 Jul 2015 20:53:06 +0100 Subject: [PATCH] compile: add line numbers to compiler and create line number table (lnotab) --- compile/compile.go | 25 ++++++++++++--- compile/compile_test.go | 37 +++++++++++++++++++++- compile/instructions.go | 59 +++++++++++++++++++++++++++++++++--- compile/instructions_test.go | 56 ++++++++++++++++++++++++++++++++++ notes.txt | 5 +-- 5 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 compile/instructions_test.go diff --git a/compile/compile.go b/compile/compile.go index 8916e0e5..0515f4c3 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -68,6 +68,7 @@ const ( type compiler struct { Code *py.Code // code being built up Filename string + Lineno int // current line number OpCodes Instructions loops loopstack SymTable *symtable.SymTable @@ -144,6 +145,11 @@ func (c *compiler) panicSyntaxErrorf(Ast ast.Ast, format string, a ...interface{ panic(err) } +// Sets Lineno from an ast node +func (c *compiler) SetLineno(node ast.Ast) { + c.Lineno = node.GetLineno() +} + // Create a new compiler object at Ast, using private for name mangling func (c *compiler) newCompilerScope(compilerScope compilerScopeType, Ast ast.Ast, private string) (newC *compiler) { newSymTable := c.SymTable.FindChild(Ast) @@ -186,6 +192,7 @@ func (c *compiler) compileAst(Ast ast.Ast, filename string, futureFlags int, don code.Freevars = append(code.Freevars, SymTable.Find(symtable.ScopeFree, symtable.DefFreeClass)...) code.Flags = c.codeFlags(SymTable) | int32(futureFlags&py.CO_COMPILER_FLAGS_MASK) valueOnStack := false + c.SetLineno(Ast) switch node := Ast.(type) { case *ast.Module: c.Stmts(c.docString(node.Body, false)) @@ -283,6 +290,7 @@ func (c *compiler) compileAst(Ast ast.Ast, filename string, futureFlags int, don code.Code = c.OpCodes.Assemble() code.Stacksize = int32(c.OpCodes.StackDepth()) code.Nlocals = int32(len(code.Varnames)) + code.Lnotab = string(c.OpCodes.Lnotab()) return nil } @@ -377,7 +385,9 @@ func (c *compiler) OpArg(Op vm.OpCode, Arg uint32) { if !Op.HAS_ARG() { panic("OpArg called with an instruction which doesn't take an Arg") } - c.OpCodes.Add(&OpArg{Op: Op, Arg: Arg}) + instr := &OpArg{Op: Op, Arg: Arg} + instr.SetLineno(c.Lineno) + c.OpCodes.Add(instr) } // Compiles an instruction without an argument @@ -385,7 +395,9 @@ func (c *compiler) Op(op vm.OpCode) { if op.HAS_ARG() { panic("Op called with an instruction which takes an Arg") } - c.OpCodes.Add(&Op{Op: op}) + instr := &Op{Op: op} + instr.SetLineno(c.Lineno) + c.OpCodes.Add(instr) } // Inserts an existing label @@ -402,14 +414,17 @@ func (c *compiler) NewLabel() *Label { // Compiles a jump instruction func (c *compiler) Jump(Op vm.OpCode, Dest *Label) { + var instr Instruction switch Op { case vm.JUMP_IF_FALSE_OR_POP, vm.JUMP_IF_TRUE_OR_POP, vm.JUMP_ABSOLUTE, vm.POP_JUMP_IF_FALSE, vm.POP_JUMP_IF_TRUE, vm.CONTINUE_LOOP: // Absolute - c.OpCodes.Add(&JumpAbs{OpArg: OpArg{Op: Op}, Dest: Dest}) + instr = &JumpAbs{OpArg: OpArg{Op: Op}, Dest: Dest} case vm.JUMP_FORWARD, vm.SETUP_WITH, vm.FOR_ITER, vm.SETUP_LOOP, vm.SETUP_EXCEPT, vm.SETUP_FINALLY: - c.OpCodes.Add(&JumpRel{OpArg: OpArg{Op: Op}, Dest: Dest}) + instr = &JumpRel{OpArg: OpArg{Op: Op}, Dest: Dest} default: panic("Jump called with non jump instruction") } + instr.SetLineno(c.Lineno) + c.OpCodes.Add(instr) } /* The test for LOCAL must come before the test for FREE in order to @@ -963,6 +978,7 @@ func (c *compiler) Stmts(stmts []ast.Stmt) { // Compile statement func (c *compiler) Stmt(stmt ast.Stmt) { + c.SetLineno(stmt) switch node := stmt.(type) { case *ast.FunctionDef: // Name Identifier @@ -1578,6 +1594,7 @@ func (c *compiler) Exprs(exprs []ast.Expr) { // Compile and expression func (c *compiler) Expr(expr ast.Expr) { + c.SetLineno(expr) switch node := expr.(type) { case *ast.BoolOp: // Op BoolOpNumber diff --git a/compile/compile_test.go b/compile/compile_test.go index 476a8a30..ad1c17a4 100644 --- a/compile/compile_test.go +++ b/compile/compile_test.go @@ -19,6 +19,39 @@ func EqString(t *testing.T, name string, a, b string) { } } +// Compact the lnotab as python3 seems to generate inefficient ones +// with spurious zero line number increments. +func lnotabCompact(t *testing.T, pxs *[]byte) { + xs := *pxs + newxs := make([]byte, 0, len(xs)) + carry := 0 + for i := 0; i < len(xs); i += 2 { + d_offset, d_lineno := xs[i], xs[i+1] + if d_lineno == 0 { + carry += int(d_offset) + continue + } + // FIXME ignoring d_offset overflow + d_offset += byte(carry) + carry = 0 + newxs = append(newxs, byte(d_offset), d_lineno) + } + // if string(newxs) != string(xs) { + // t.Logf("Compacted\n% x\n% x", xs, newxs) + // } + *pxs = newxs +} + +func EqLnotab(t *testing.T, name string, aStr, bStr string) { + a := []byte(aStr) + b := []byte(bStr) + lnotabCompact(t, &a) + lnotabCompact(t, &b) + if string(a) != string(b) { + t.Errorf("%s want % x, got % x", name, a, b) + } +} + func EqInt32(t *testing.T, name string, a, b int32) { if a != b { t.Errorf("%s want %d, got %d", name, a, b) @@ -105,7 +138,9 @@ func EqCode(t *testing.T, name string, a, b *py.Code) { EqCodeCode(t, name+": Code", a.Code, b.Code) EqString(t, name+": Filename", a.Filename, b.Filename) EqString(t, name+": Name", a.Name, b.Name) - // FIXME EqString(t, name+": Lnotab", a.Lnotab, b.Lnotab) + // The Lnotabs are mostly the same but not entirely + // So it is probably not profitable to test them exactly + // EqLnotab(t, name+": Lnotab", a.Lnotab, b.Lnotab) // []string EqStrings(t, name+": Names", a.Names, b.Names) diff --git a/compile/instructions.go b/compile/instructions.go index d2dcb1fb..0392f71a 100644 --- a/compile/instructions.go +++ b/compile/instructions.go @@ -1,8 +1,6 @@ package compile -import ( - "github.com/ncw/gpython/vm" -) +import "github.com/ncw/gpython/vm" // FIXME detect if label is not in the instruction stream by setting // Pos to 0xFFFF say by default, ie we made a label but forgot to add @@ -300,6 +298,8 @@ func (is Instructions) StackDepth() int { type Instruction interface { Pos() uint32 Number() int + Lineno() int + SetLineno(int) SetPos(int, uint32) bool Size() uint32 Output() []byte @@ -312,8 +312,9 @@ type Resolver interface { // Position type pos struct { - n uint32 - p uint32 + n uint32 + p uint32 + lineno int } // Read instruction number @@ -326,6 +327,16 @@ func (p *pos) Pos() uint32 { return p.p } +// Read lineno +func (p *pos) Lineno() int { + return p.lineno +} + +// Set lineno +func (p *pos) SetLineno(lineno int) { + p.lineno = lineno +} + // Set Position - returns changed func (p *pos) SetPos(number int, newPos uint32) bool { p.n = uint32(number) @@ -439,3 +450,41 @@ func (o *JumpRel) Resolve() { panic("FIXME compile: JUMP_FOWARDS size changed") } } + +// Creates the lnotab from the instruction stream +// +// See Objects/lnotab_notes.txt for the description of the line number table. +func (is Instructions) Lnotab() []byte { + var lnotab []byte + old_offset := uint32(0) + old_lineno := 1 + for _, instr := range is { + if instr.Size() == 0 { + continue + } + lineno := instr.Lineno() + offset := instr.Pos() + d_lineno := lineno - old_lineno + if d_lineno <= 0 { + continue + } + d_bytecode := offset - old_offset + for d_bytecode > 255 { + lnotab = append(lnotab, 255, 0) + d_bytecode -= 255 + } + for d_lineno > 255 { + lnotab = append(lnotab, byte(d_bytecode), 255) + d_bytecode = 0 + d_lineno -= 255 + } + if d_bytecode > 0 { + lnotab = append(lnotab, byte(d_bytecode), byte(d_lineno)) + } else { /* First line of a block; def stmt, etc. */ + lnotab = append(lnotab, 0, byte(d_lineno)) + } + old_lineno = lineno + old_offset = offset + } + return lnotab +} diff --git a/compile/instructions_test.go b/compile/instructions_test.go new file mode 100644 index 00000000..cda00239 --- /dev/null +++ b/compile/instructions_test.go @@ -0,0 +1,56 @@ +package compile + +import ( + "bytes" + "testing" +) + +func TestLnotab(t *testing.T) { + for i, test := range []struct { + instrs Instructions + want []byte + }{ + { + instrs: Instructions{}, + want: []byte{}, + }, + { + instrs: Instructions{ + &Op{pos: pos{n: 1, p: 10, lineno: 1}}, + &Op{pos: pos{n: 0, p: 10, lineno: 0}}, + &Op{pos: pos{n: 1, p: 102, lineno: 1}}, + }, + want: []byte{}, + }, + { + instrs: Instructions{ + &Op{pos: pos{n: 1, p: 0, lineno: 1}}, + &Op{pos: pos{n: 1, p: 1, lineno: 2}}, + &Op{pos: pos{n: 1, p: 2, lineno: 3}}, + }, + want: []byte{1, 1, 1, 1}, + }, + { + // Example from lnotab.txt + instrs: Instructions{ + &Op{pos: pos{n: 1, p: 0, lineno: 1}}, + &Op{pos: pos{n: 1, p: 6, lineno: 2}}, + &Op{pos: pos{n: 1, p: 50, lineno: 7}}, + &Op{pos: pos{n: 1, p: 350, lineno: 307}}, + &Op{pos: pos{n: 1, p: 361, lineno: 308}}, + }, + want: []byte{ + 6, 1, + 44, 5, + 255, 0, + 45, 255, + 0, 45, + 11, 1}, + }, + } { + got := test.instrs.Lnotab() + if bytes.Compare(test.want, got) != 0 { + t.Errorf("%d: want %d got %d", i, test.want, got) + } + } +} diff --git a/notes.txt b/notes.txt index a4ef2bdc..6092f6a1 100644 --- a/notes.txt +++ b/notes.txt @@ -18,6 +18,7 @@ Things to do before release =========================== * Line numbers + * frame.Lasti is pointing past the instruction which puts tracebacks out * Subclass builtins * pygen @@ -39,10 +40,6 @@ Failes on /usr/lib/python3/dist-packages/mpmath/usertools.py Limitations & Missing parts =========================== * string keys only in dictionaries - * line numbers missing in SyntaxErrors - * now need to show them when printing SyntaxErrors - * line numbers missing in Tracebacks - * lnotab etc * \N{...} escapes not implemented * lots of builtins still to implement * FIXME eq && ne should throw an error for a type which doesn' have eq implemented