Pickle and unpickle type trees

Lots of other changes to make positions work out everywhere.
One important change is that now trees can be shared, just
as types can. This change improves memory requirements (a bit)
and also makes positions in shared trees more robust.

docs/syntax-summary.txt

@@ -160,7 +160,7 @@ grammar.
   SimpleExpr1       ::=  Literal
                       |  Path
                       |  `_'
-                      |  `(' ExprsInParens `)'                                  Parens(exprs)
+                      |  `(' ExprsInParens2 `)'                                  Parens(exprs)
                       |  SimpleExpr `.' id                                      Select(expr, id)
                       |  SimpleExpr (TypeArgs | NamedTypeArgs)                  TypeApply(expr, args)
                       |  SimpleExpr1 ArgumentExprs                              Apply(expr, args)
@@ -210,7 +210,7 @@ grammar.
                       |  SimplePattern1 `.' id
   PatVar            ::=  varid
                       |  `_'
-  Patterns          ::=  Pattern [`,' Pattern]
+  Patterns          ::=  Pattern {`,' Pattern}
   ArgumentPatterns  ::=  `(' [Patterns] `)'                                     Apply(fn, pats)
                       |  `(' [Patterns `,'] Pattern2 `:' `_' `*' ')
 

src/dotty/tools/dotc/ast/tpd.scala

@@ -133,14 +133,18 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
   def OrTypeTree(left: Tree, right: Tree)(implicit ctx: Context): OrTypeTree =
     ta.assignType(untpd.OrTypeTree(left, right), left, right)
 
-  // RefinedTypeTree is missing, handled specially in Typer and Unpickler.
+  def RefinedTypeTree(parent: Tree, refinements: List[Tree], refineCls: ClassSymbol)(implicit ctx: Context): Tree =
+    ta.assignType(untpd.RefinedTypeTree(parent, refinements), parent, refinements, refineCls)
 
   def AppliedTypeTree(tycon: Tree, args: List[Tree])(implicit ctx: Context): AppliedTypeTree =
     ta.assignType(untpd.AppliedTypeTree(tycon, args), tycon, args)
 
   def ByNameTypeTree(result: Tree)(implicit ctx: Context): ByNameTypeTree =
     ta.assignType(untpd.ByNameTypeTree(result), result)
 
+  def PolyTypeTree(tparams: List[TypeDef], body: Tree)(implicit ctx: Context): PolyTypeTree =
+    ta.assignType(untpd.PolyTypeTree(tparams, body), tparams, body)
+
   def TypeBoundsTree(lo: Tree, hi: Tree)(implicit ctx: Context): TypeBoundsTree =
     ta.assignType(untpd.TypeBoundsTree(lo, hi), lo, hi)
 
@@ -306,8 +310,8 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
       case _ =>
         false
     }
-    typeIsElidable || 
-    tp.symbol.is(JavaStatic) || 
+    typeIsElidable ||
+    tp.symbol.is(JavaStatic) ||
     tp.symbol.hasAnnotation(defn.ScalaStaticAnnot)
   }
 

src/dotty/tools/dotc/core/Definitions.scala

@@ -633,9 +633,9 @@ class Definitions {
     name.startsWith(prefix) && name.drop(prefix.length).forall(_.isDigit)
   }
 
-  def isBottomClass(cls: Symbol) = 
+  def isBottomClass(cls: Symbol) =
     cls == NothingClass || cls == NullClass
-  def isBottomType(tp: Type) = 
+  def isBottomType(tp: Type) =
     tp.derivesFrom(NothingClass) || tp.derivesFrom(NullClass)
 
   def isFunctionClass(cls: Symbol) = isVarArityClass(cls, tpnme.Function)
@@ -785,8 +785,8 @@ class Definitions {
     if (!_isInitialized) {
       // force initialization of every symbol that is synthesized or hijacked by the compiler
       val forced = syntheticCoreClasses ++ syntheticCoreMethods ++ ScalaValueClasses()
-      
-      // Enter all symbols from the scalaShadowing package in the scala package 
+
+      // Enter all symbols from the scalaShadowing package in the scala package
       for (m <- ScalaShadowingPackageClass.info.decls)
         ScalaPackageClass.enter(m)
 

src/dotty/tools/dotc/core/tasty/PositionPickler.scala

@@ -13,29 +13,14 @@ import collection.mutable
 import TastyBuffer._
 import util.Positions._
 
-class PositionPickler(pickler: TastyPickler, addrsOfTree: tpd.Tree => List[Addr]) {
+class PositionPickler(pickler: TastyPickler, addrOfTree: tpd.Tree => Option[Addr]) {
   val buf = new TastyBuffer(5000)
   pickler.newSection("Positions", buf)
   import buf._
   import ast.tpd._
 
   private val remainingAddrs = new java.util.IdentityHashMap[Tree, Iterator[Addr]]
 
-  def nextTreeAddr(tree: Tree): Option[Addr] = remainingAddrs.get(tree) match {
-    case null =>
-      addrsOfTree(tree) match {
-        case Nil =>
-          None
-        case addr :: Nil =>
-          Some(addr)
-        case addrs =>
-          remainingAddrs.put(tree, addrs.iterator)
-          nextTreeAddr(tree)
-      }
-    case it: Iterator[_] =>
-      if (it.hasNext) Some(it.next) else None
-  }
-
   def header(addrDelta: Int, hasStartDelta: Boolean, hasEndDelta: Boolean, hasPoint: Boolean) = {
     def toInt(b: Boolean) = if (b) 1 else 0
     (addrDelta << 3) | (toInt(hasStartDelta) << 2) | (toInt(hasEndDelta) << 1) | toInt(hasPoint)
@@ -60,16 +45,15 @@ class PositionPickler(pickler: TastyPickler, addrsOfTree: tpd.Tree => List[Addr]
      */
     def alwaysNeedsPos(x: Positioned) = x match {
       case _: WithLazyField[_]            // initialPos is inaccurate for trees with lazy field
-         | _: Trees.PackageDef[_] => true // package defs might be split into several Tasty files 
-      case x: Trees.Tree[_] => x.isType   // types are unpickled as TypeTrees, so child positions are not available
+         | _: Trees.PackageDef[_] => true // package defs might be split into several Tasty files
       case _ => false
     }
 
     def traverse(x: Any): Unit = x match {
       case x: Tree @unchecked =>
         val pos = if (x.isInstanceOf[MemberDef]) x.pos else x.pos.toSynthetic
         if (pos.exists && (pos != x.initialPos.toSynthetic || alwaysNeedsPos(x))) {
-          nextTreeAddr(x) match {
+          addrOfTree(x) match {
             case Some(addr) =>
               //println(i"pickling $x with $pos at $addr")
               pickleDeltas(addr.index, pos)
@@ -79,13 +63,15 @@ class PositionPickler(pickler: TastyPickler, addrsOfTree: tpd.Tree => List[Addr]
         }
         //else if (x.pos.exists) println(i"skipping $x")
         x match {
-          case x: MemberDef @unchecked => 
+          case x: MemberDef @unchecked =>
             for (ann <- x.symbol.annotations) traverse(ann.tree)
           case _ =>
         }
         traverse(x.productIterator)
       case xs: TraversableOnce[_] =>
         xs.foreach(traverse)
+      case x: Annotation =>
+        traverse(x.tree)
       case _ =>
     }
     traverse(roots)

src/dotty/tools/dotc/core/tasty/TastyFormat.scala

@@ -71,7 +71,7 @@ Standard-Section: "ASTs" TopLevelStat*
 
   Term          = Path
                   Application
-                  IDENT                 NameRef Type     // used when ident’s type is not a TermRef
+                  IDENT                 NameRef Type     // used when term ident’s type is not a TermRef
                   SELECT                possiblySigned_NameRef qual_Term
                   NEW                   cls_Type
                   SUPER          Length this_Term mixinTrait_Type?
@@ -89,6 +89,17 @@ Standard-Section: "ASTs" TopLevelStat*
                   BIND           Length boundName_NameRef patType_Type pat_Term
                   ALTERNATIVE    Length alt_Term*
                   UNAPPLY        Length fun_Term ImplicitArg* pat_Type pat_Term*
+                  IDENTtpt              NameRef Type     // used when type ident's type is not a TypeRef
+                  SELECTtpt             NameRef qual_Term
+                  SINGLETONtpt          Path
+                  REFINDtpt      Length underlying_Term refinement_Stat*
+                  APPLIEDtpt     Length tycon_Term arg_Term*
+                  POLYtpt        Length TypeParam* body_Term
+                  TYPEBOUNDStpt  Length low_Term high_Term
+                  ANNOTATEDtpt   Length underlying_Term fullAnnotation_Term
+                  ANDtpt         Length left_Term right_Term
+                  ORtpt          Length left_Term right_Term
+                  BYNAMEtpt             underlying_Term
                   EMPTYTREE
                   SHARED                term_ASTRef
   Application   = APPLY          Length fn_Term arg_Term*
@@ -133,7 +144,7 @@ Standard-Section: "ASTs" TopLevelStat*
                   APPLIEDtype    Length tycon_Type arg_Type*
                   TYPEBOUNDS     Length low_Type high_Type
                   TYPEALIAS      Length alias_Type (COVARIANT | CONTRAVARIANT)?
-                  ANNOTATED      Length underlying_Type fullAnnotation_Term
+                  ANNOTATEDtype  Length underlying_Type fullAnnotation_Term
                   ANDtype        Length left_Type right_Type
                   ORtype         Length left_Type right_Type
                   BIND           Length boundName_NameRef bounds_Type
@@ -271,19 +282,23 @@ object TastyFormat {
   final val CLASSconst = 97
   final val ENUMconst = 98
   final val BYNAMEtype = 99
-  final val NEW = 100
-  final val IMPLICITarg = 101
-  final val PRIVATEqualified = 102
-  final val PROTECTEDqualified = 103
-  final val RECtype = 104
+  final val BYNAMEtpt = 100
+  final val NEW = 101
+  final val IMPLICITarg = 102
+  final val PRIVATEqualified = 103
+  final val PROTECTEDqualified = 104
+  final val RECtype = 105
+  final val SINGLETONtpt = 106
 
   final val IDENT = 112
-  final val SELECT = 113
-  final val TERMREFsymbol = 114
-  final val TERMREF = 115
-  final val TYPEREFsymbol = 116
-  final val TYPEREF = 117
-  final val SELFDEF = 118
+  final val IDENTtpt = 113
+  final val SELECT = 114
+  final val SELECTtpt = 115
+  final val TERMREFsymbol = 116
+  final val TERMREF = 117
+  final val TYPEREFsymbol = 118
+  final val TYPEREF = 119
+  final val SELFDEF = 120
 
   final val PACKAGE = 128
   final val VALDEF = 129
@@ -293,39 +308,44 @@ object TastyFormat {
   final val TYPEPARAM = 133
   final val PARAMS = 134
   final val PARAM = 136
-
-  final val APPLY = 139
-  final val TYPEAPPLY = 140
-
-  final val TYPED = 143
-  final val NAMEDARG = 144
-  final val ASSIGN = 145
-  final val BLOCK = 146
-  final val IF = 147
-  final val LAMBDA = 148
-  final val MATCH = 149
-  final val RETURN = 150
-  final val TRY = 151
-  final val INLINED = 152
-  final val REPEATED = 153
-  final val BIND = 154
-  final val ALTERNATIVE = 155
-  final val UNAPPLY = 156
-  final val ANNOTATED = 157
-  final val CASEDEF = 158
-  final val TEMPLATE = 160
-  final val SUPER = 163
-  final val SUPERtype = 166
-  final val REFINEDtype = 167
-  final val APPLIEDtype = 168
-  final val TYPEBOUNDS = 169
-  final val TYPEALIAS = 170
-  final val ANDtype = 171
-  final val ORtype = 172
-  final val METHODtype = 174
-  final val POLYtype = 175
-  final val PARAMtype = 176
-  final val ANNOTATION = 177
+  final val APPLY = 137
+  final val TYPEAPPLY = 138
+  final val TYPED = 139
+  final val NAMEDARG = 140
+  final val ASSIGN = 141
+  final val BLOCK = 142
+  final val IF = 143
+  final val LAMBDA = 144
+  final val MATCH = 145
+  final val RETURN = 146
+  final val TRY = 147
+  final val INLINED = 148
+  final val REPEATED = 149
+  final val BIND = 150
+  final val ALTERNATIVE = 151
+  final val UNAPPLY = 152
+  final val ANNOTATEDtype = 153
+  final val ANNOTATEDtpt = 154
+  final val CASEDEF = 155
+  final val TEMPLATE = 156
+  final val SUPER = 157
+  final val SUPERtype = 158
+  final val REFINEDtype = 159
+  final val REFINEDtpt = 160
+  final val APPLIEDtype = 161
+  final val APPLIEDtpt = 162
+  final val TYPEBOUNDS = 163
+  final val TYPEBOUNDStpt = 164
+  final val TYPEALIAS = 165
+  final val ANDtype = 166
+  final val ANDtpt = 167
+  final val ORtype = 168
+  final val ORtpt = 169
+  final val METHODtype = 170
+  final val POLYtype = 171
+  final val POLYtpt = 172
+  final val PARAMtype = 173
+  final val ANNOTATION = 174
 
   final val firstSimpleTreeTag = UNITconst
   final val firstNatTreeTag = SHARED
@@ -367,7 +387,22 @@ object TastyFormat {
        | PRIVATEqualified
        | PROTECTEDqualified => true
     case _ => false
-   }
+  }
+
+  def isTypeTreeTag(tag: Int) = tag match {
+    case IDENTtpt
+       | SELECTtpt
+       | SINGLETONtpt
+       | REFINEDtpt
+       | APPLIEDtpt
+       | POLYtpt
+       | TYPEBOUNDStpt
+       | ANNOTATEDtpt
+       | ANDtpt
+       | ORtpt
+       | BYNAMEtpt => true
+    case _ => false
+  }
 
   def nameTagToString(tag: Int): String = tag match {
     case UTF8 => "UTF8"
@@ -429,7 +464,9 @@ object TastyFormat {
     case RECtype => "RECtype"
 
     case IDENT => "IDENT"
+    case IDENTtpt => "IDENTtpt"
     case SELECT => "SELECT"
+    case SELECTtpt => "SELECTtpt"
     case TERMREFsymbol => "TERMREFsymbol"
     case TERMREF => "TERMREF"
     case TYPEREFsymbol => "TYPEREFsymbol"
@@ -462,7 +499,8 @@ object TastyFormat {
     case BIND => "BIND"
     case ALTERNATIVE => "ALTERNATIVE"
     case UNAPPLY => "UNAPPLY"
-    case ANNOTATED => "ANNOTATED"
+    case ANNOTATEDtype => "ANNOTATEDtype"
+    case ANNOTATEDtpt => "ANNOTATEDtpt"
     case CASEDEF => "CASEDEF"
     case IMPLICITarg => "IMPLICITarg"
     case TEMPLATE => "TEMPLATE"
@@ -471,15 +509,23 @@ object TastyFormat {
     case SUPER => "SUPER"
     case CLASSconst => "CLASSconst"
     case ENUMconst => "ENUMconst"
+    case SINGLETONtpt => "SINGLETONtpt"
     case SUPERtype => "SUPERtype"
     case REFINEDtype => "REFINEDtype"
+    case REFINEDtpt => "REFINEDtpt"
     case APPLIEDtype => "APPLIEDtype"
+    case APPLIEDtpt => "APPLIEDtpt"
     case TYPEBOUNDS => "TYPEBOUNDS"
+    case TYPEBOUNDStpt => "TYPEBOUNDStpt"
     case TYPEALIAS => "TYPEALIAS"
     case ANDtype => "ANDtype"
+    case ANDtpt => "ANDtpt"
     case ORtype => "ORtype"
+    case ORtpt => "ORtpt"
     case BYNAMEtype => "BYNAMEtype"
+    case BYNAMEtpt => "BYNAMEtpt"
     case POLYtype => "POLYtype"
+    case POLYtpt => "POLYtpt"
     case METHODtype => "METHODtype"
     case PARAMtype => "PARAMtype"
     case ANNOTATION => "ANNOTATION"

src/dotty/tools/dotc/core/tasty/TastyPickler.scala

@@ -61,7 +61,7 @@ class TastyPickler {
    * Note that a tree can have several addresses, if it is shared,
    * i.e. accessible from different paths. Any such sharing is undone by pickling.
    */
-  var addrsOfTree: tpd.Tree => List[Addr] = (_ => Nil)
+  var addrOfTree: tpd.Tree => Option[Addr] = (_ => None)
 
   /**
    * Addresses in TASTY file of symbols, stored by pickling.