TI1220 2012-2013Concepts of Programming Languages

Eelco Visser / TU Delft

Lecture 9: Parsing & Interpretation

Syntax and SemanticsNames, Bindings, and Scopes Storage Data TypesFunctional ProgrammingFirst-class FunctionsPolymorphism

Traits & Type ParameterizationParsing and InterpretationData Abstraction / Modular ProgrammingFunctional Programming ReduxConcurrencyConcurrent ProgrammingDomain-Specific Languages

Quarter 3

Quarter 4

Basics ofScalaJavaScriptC

eval(parse(exp)) == value

parser

interpreter

Syntax + Semantics

Parsing

“The syntax of a programming language is the form of its expressions, statements,

and program units.”Sebesta Ch3

let y = 3 in (let inc = fun(x){ x + y } in (let y = 11 in (inc 2)))

let g = fun(f){ f(3) } in (let h = fun(x){ x + 5 } in (g h))

Example Language

Exp = Exp '+' Exp1 // addition | Exp1Exp1 = Exp1 Exp0 // function application | Exp0Exp0 = '(' Exp ') // parentheses | 'fun' '(' ident ')' '{' // function literal Exp '}' | 'let' ident '=' Exp // let binding 'in' Exp0 | ident // identifier | number // number

Context-free Grammar

sealed abstract class Expcase class Add(lhs: Exp, rhs: Exp) extends Expcase class App(funExpr: Exp, arg: Exp) extends Expcase class Fun(param: Symbol, body: Exp) extends Expcase class Let(name: Symbol, bound: Exp, body: Exp) extends Expcase class Id(name: Symbol) extends Expcase class Num(n: Int) extends Exp

Exp = Exp '+' Exp1 | Exp1Exp1 = Exp1 Exp0 | Exp0Exp0 = '(' Exp ') | 'fun' '(' ident ')' '{' Exp '}' | 'let' ident '=' Exp 'in' Exp0 | ident | number

From Concrete to Abstract Syntax

Sentence AST

1 + 2 Add(Num(1),Num(2))

foo bar baz App(App(Id('foo), Id('bar)), Id('baz))

fun(foo){ bar baz } Fun('foo, App(Id('bar), Id('baz)))

fun(x){ x + 1 } Fun('x, Add(Id('x), Num(1)))

let inc = fun(x){ x + 1 } in (inc 3)

Let('inc, Fun('x, Add(Id('x), Num(1))), App(Id('inc), Num(3)))

From Concrete to Abstract Syntax

def parse(text: String): Exp

Parser

Approaches to constructing parsers

★ roll your own (recursive descent)

★ parser generator

★ parser combinators

Parser Combinators

Arithmetic Expressions

(1 + 4) * 7 / 34 - 3

expr ::= term { "+" term | "-" term }.

term ::= factor {"*" factor | "/" factor }.

factor ::= floatingPointNumber | "(" expr ")".

import scala.util.parsing.combinator._class Arith extends JavaTokenParsers { def expr: Parser[Any] = term~rep("+"~term | "-"~term) def term: Parser[Any] = factor~rep("*"~factor | "/"~factor) def factor: Parser[Any] = floatingPointNumber | "("~expr~")"}

expr ::= term { "+" term | "-" term }.

term ::= factor {"*" factor | "/" factor }.

factor ::= floatingPointNumber | "(" expr ")".

Parser combinators★ Sequential composition: p ~ q★ Repetition: rep( p )★ Alternative: p | q

Full API: http://www.scala-lang.org/api/current/index.html#scala.reflect.macros.Parsers

(1 + 4) * 7

[[1.12] parsed: (((((~((1~List())~List((+~(4~List())))))~))~List((*~7)))~List())]

sealed abstract class Expcase class Num(n: Double) extends Expcase class Op(op: String, e: Exp) extends Expcase class OpList(e: Exp, es: List[Exp]) extends Exp

import scala.util.parsing.combinator._class Arith extends JavaTokenParsers { def expr: Parser[Exp] = term ~ rep( "+" ~> term ^^ { case e => Op("+", e) } ) ^^ { case e~es => OpList(e, es) } def term: Parser[Exp] = factor ~ rep( "*" ~> factor ^^ { case e => Op("*", e) } ) ^^ { case e~es => OpList(e, es) } def factor: Parser[Exp] = floatingPointNumber ^^ { case n => Num(n.toDouble) } | "(" ~> expr <~ ")" }

object ParseExpr extends Arith { def parse(exp: String): Exp = parseAll(expr, exp).get}

Constructing ASTs

(1 + 3)

OpList(OpList(OpList(OpList(Num(1.0),List()), List(Op(+,OpList(Num(3.0),List())))),List()),List())

sealed abstract class Expcase class Num(n: Double) extends Expcase class BinOp(op: String, l: Exp, r: Exp) extends Exp

import scala.util.parsing.combinator._class Arith extends JavaTokenParsers { def expr: Parser[Exp] = expr ~ ("+" ~> term) ^^ { case e~t => BinOp("+", e, t) } | term def term: Parser[Exp] = term ~ ("*" ~> factor) ^^ { case e~t => BinOp("*", e, t) } | factor def factor: Parser[Exp] = floatingPointNumber ^^ { case n => Num(n.toDouble) } | "(" ~> expr <~ ")" }

(1 + 3)

Binary Productions

sealed abstract class Expcase class Num(n: Double) extends Expcase class BinOp(op: String, l: Exp, r: Exp) extends Exp

import scala.util.parsing.combinator._class Arith extends JavaTokenParsers { def expr: Parser[Exp] = expr ~ ("+" ~> term) ^^ { case e~t => BinOp("+", e, t) } | term def term: Parser[Exp] = term ~ ("*" ~> factor) ^^ { case e~t => BinOp("*", e, t) } | factor def factor: Parser[Exp] = floatingPointNumber ^^ { case n => Num(n.toDouble) } | "(" ~> expr <~ ")" }

(1 + 3) Status: RunningFailureTest score: 0/-1

Binary Productions

sealed abstract class Expcase class Num(n: Double) extends Expcase class BinOp(op: String, l: Exp, r: Exp) extends Exp

import scala.util.parsing.combinator._class Arith extends JavaTokenParsers with PackratParsers { lazy val expr: PackratParser[Exp] = expr ~ ("+" ~> term) ^^ { case e~t => BinOp("+", e, t) } | term lazy val term: PackratParser[Exp] = factor ~ ("*" ~> factor) ^^ { case e~t => BinOp("*", e, t) } | factor lazy val factor: PackratParser[Exp] = floatingPointNumber ^^ { case n => Num(n.toDouble) } | "(" ~> expr <~ ")" }

(1 + 4) * 7BinOp(*, BinOp(+,Num(1.0),Num(4.0)), Num(7.0))

Packrat Parser

{ "address book": { "name": "John Smith", "address": { "street": "10 Market Street", "city" : "San Francisco, CA", "zip" : 94111 }, "phone numbers": [ "408 3384238", "408 1116892" ] }}

Map( address book -> Map( name -> John Smith, address -> Map( street -> 10 Market Street, city -> San Francisco, CA, zip -> 94111), phone numbers -> List(408 3384238, 408 1116892) ))

Parsing JSON

import scala.util.parsing.combinator._class JSON1 extends JavaTokenParsers { def obj: Parser[Map[String, Any]] = "{" ~> repsep(member, ",") <~ "}" ^^ (Map() ++ _) def arr: Parser[List[Any]] = "[" ~> repsep(value, ",") <~ "]" def member: Parser[(String, Any)] = stringLiteral ~ ":" ~ value ^^ { case name ~ ":" ~ value => (name, value) } def value: Parser[Any] = ( obj | arr | stringLiteral | floatingPointNumber ^^ (_.toDouble) | "null" ^^ (x => null) | "true" ^^ (x => true) | "false" ^^ (x => false))}

JSON Parser

Parser Combinators

✦ "..." literal

✦ "...".r regular expression

✦ P~Q sequential composition

✦ P <~ Q, P ~> Q sequential composition; keep left/right only

✦ P | Q alternative

✦ opt(P) option

✦ rep(P) repetition

✦ repsep(P, Q) interleaved repetition

✦ P ˆˆ f result conversion

Example Language

object ExpParser extends JavaTokenParsers with PackratParsers { lazy val exp: PackratParser[Exp] = (exp <~ "+") ~ exp1 ^^ { case lhs~rhs => Add(lhs, rhs) } | exp1

lazy val exp1: PackratParser[Exp] = (exp1 ~ exp0) ^^ { case lhs~rhs => App(lhs, rhs) } | exp0 lazy val exp0: PackratParser[Exp] = number | identifier | function | letBinding | "(" ~> exp <~ ")" // ... def parse(text: String) = parseAll(exp, text)}

def number: PackratParser[Exp] = wholeNumber ^^ { x => Num(x.toInt) }

Numbers

def function: PackratParser[Exp] = ("fun" ~ "(") ~> ident ~ ((")" ~ "{") ~> (exp <~ "}")) ^^ { case x~e => Fun(Symbol(x), e) }

Function Literal

def letBinding: PackratParser[Exp] = (("let" ~> ident) ~ ("=" ~> exp)) ~ ("in" ~> exp0) ^^ { case x~e1~e2 => Let(Symbol(x), e1, e2) }

Let Binding

Exercise

Reserved Words

def identifier: PackratParser[Exp] = not(reserved) ~> ident ^^ { x => Id(Symbol(x)) } def reserved: PackratParser[String] = "let\\b".r | "in\\b".r | "fun\\b".r

Implementing Parser Combinators

package scala.util.parsing.combinatortrait Parsers { type Parser[T] = Input => ParseResult[T]

type Input = Reader[Elem]

type Elem

// definition of parser combinators

}

Parser Trait

sealed abstract class ParseResult[+T]

case class Success[T](result: T, in: Input) extends ParseResult[T]

case class Failure(msg: String, in: Input) extends ParseResult[Nothing]

Parse Result

abstract class Parser[+T] extends (Input => ParseResult[T]){ p => // An unspecified method that defines // the behavior of this parser. def apply(in: Input): ParseResult[T] def ~ // ... def | // ... // ...}

Parser

def elem(kind: String, p: Elem => Boolean) = new Parser[Elem] { def apply(in: Input) = if (p(in.first)) Success(in.first, in.rest) else Failure(kind + " expected", in) }

Single Element Parser

abstract class Parser[+T] //... { p => //... def ~ [U](q: => Parser[U]) = new Parser[T~U] { def apply(in: Input) = p(in) match { case Success(x, in1) => q(in1) match { case Success(y, in2) => Success(new ~(x, y), in2) case failure => failure } case failure => failure }}

Sequential Composition

def | (q: => Parser[T]) = new Parser[T] { def apply(in: Input) = p(in) match { case s1 @ Success(_, _) => s1 case failure => q(in) }}

Alternative Composition

def ^^ [U](f: T => U): Parser[U] = new Parser[U] { def apply(in: Input) = p(in) match { case Success(x, in1) => Success(f(x), in1) case failure => failure }}

Result Conversion

Chapter 33: more combinators

Interpreting

“The semantics of a programming language is the meaning of its expressions,

statements, and program units.”Sebesta Ch3

Static Semantics: (context-senstive) restriction of the set of valid programs

Dynamic Semantics: run-time behaviour of a program

def eval(exp: Exp): Value

dynamic behaviour of Exp programs

Calculator

def testCalc { expect(numV(48)) { eval(parse(""" 42 + 2 + 4 """)) }}

sealed abstract class Expcase class Num(n: Int) extends Expcase class Add(lhs: Exp, rhs: Exp) extends Exp

sealed abstract class Valuecase class numV(n: Int) extends Value

def plus(l: Value, r: Value) = l match { case numV(n) => r match { case numV(m) => numV(n + m) case _ => sys.error("Cannot add non-number") }}

def eval(exp: Exp): Value = exp match { case Num(v) => numV(v) case Add(l, r) => plus(eval(l), eval(r))}

Calculator

Let Bindingdef testLet { expect(numV(42)) { eval(parse(""" let x = 40 in (x + 2) """)) }}

def testLetShadow { expect(numV(?)) { eval(parse(""" let x = 40 in (let y = 4 in (let x = 3 in (x + y))) """)) }}

sealed abstract class Envcase class mtEnv() extends Envcase class bind(boundName: Symbol, boundValue: Value, rest: Env) extends Env def lookup(name: Symbol, env: Env): Value = env match { case mtEnv() => sys.error("Lookup of " + name + " in empty environment") case bind(boundName, boundValue, rest) => if(boundName == name) boundValue else lookup(name, rest)}

Environment

def eval(exp: Exp): Value = eval(exp, mtEnv()) def eval(exp: Exp, env: Env): Value = exp match { case Num(v) => numV(v) case Add(l, r) => plus(eval(l, env), eval(r, env)) case Id(name) => lookup(name, env) case Let(name, e1, e2) => eval(e2, bind(name, eval(e1, env), env))}

Evaluating Let Bindings

def testFun1 { expect(numV(5)) { eval(parse(""" let inc = fun(x){ x + 1 } in (inc 4) """)) }}

def testFun2 { expect(numV(5)) { eval(parse(""" let y = 3 in (let inc = fun(x){ x + y } in (let y = 11 in (inc 2))) """)) }}

Functions

sealed abstract class Valuecase class numV(n: Int) extends Valuecase class funV(param: Symbol, body: Exp) extends Value def eval(exp: Exp, env: Env): Value = exp match { case Num(v) => numV(v) case Add(l, r) => plus(eval(l, env), eval(r, env)) case Id(name) => lookup(name, env) case Let(name, e1, e2) => eval(e2, bind(name, eval(e1, env), env))

case Fun(name, body) => funV(name, body) case App(fun, arg) => eval(fun, env) match { case funV(name, body) => eval(body, bind(name, eval(arg, env), env)) case _ => sys.error("Function expected") }}

Functions

sealed abstract class Valuecase class numV(n: Int) extends Valuecase class funV(param: Symbol, body: Exp) extends Value def eval(exp: Exp, env: Env): Value = exp match { case Num(v) => numV(v) case Add(l, r) => plus(eval(l, env), eval(r, env)) case Id(name) => lookup(name, env) case Let(name, e1, e2) => eval(e2, bind(name, eval(e1, env), env))

case Fun(name, body) => funV(name, body) case App(fun, arg) => eval(fun, env) match { case funV(name, body) => eval(body, bind(name, eval(arg, env), env)) case _ => sys.error("Function expected") }}

FunctionsDynamic Scoping

sealed abstract class Valuecase class numV(n: Int) extends Valuecase class closureV(param: Symbol, body: Exp, env: Env) extends Value

def eval(exp: Exp, env: Env): Value = exp match { case Num(v) => numV(v) case Add(l, r) => plus(eval(l, env), eval(r, env)) case Id(name) => lookup(name, env) case Let(name, e1, e2) => eval(e2, bind(name, eval(e1, env), env))

case Fun(name, body) => closureV(name, body, env)

case App(fun, arg) => eval(fun, env) match { case closureV(name, body, env2) => eval(body, bind(name, eval(arg, env), env2)) case _ => sys.error("Closure expected") }

} Functions with Closures

Reading & Programming in Week 9

Reading

Sebesta Chapter 4: Lexical and Syntactic Analysis

Scala Chapter 33: Combinator Parsing

Week 10: Components

WebLab: Graded Assignment 2: (deadline 14 May 2013, 23:59) Graded Assignment 3: (deadline 31 May 2013, 23:59)