better string escapes, iterators

examples/iterator.cxpr

@@ -0,0 +1,36 @@
+# this function returns an iterator when called
+fn count_by(delta, limit) {
+    let counter = 0;
+    return fn() {
+        if counter >= limit {
+            return nil;
+        }
+        let prev_value = counter;
+        counter += delta;
+        return prev_value;
+    };
+}
+
+# counter is an iterator
+# iterators are functions that:
+#  - take no arguments
+#  - return nil once done
+#  - once returned nil once, must do so for all subsequent calls
+# the interpreter only checks the first requirement
+let counter = count_by(2, 5);
+println(counter()); # 0
+println(counter()); # 2
+println(counter()); # 4
+# println(counter()); # nil
+# println(counter()); # nil
+
+println("counting by 3s up to 20:");
+for n: count_by(3, 20) {
+    println(n);
+}
+
+println("counting by 7s up to 40:");
+for n: count_by(7, 40) {
+    println(n);
+}
+

src/bin/main.rs

@@ -1,4 +1,4 @@
-use std::{rc::Rc, cell::RefCell, fs, panic::{self, PanicInfo}, thread::Thread};
+use std::{rc::Rc, cell::RefCell, fs, panic::{self, PanicInfo}};
 
 use backtrace::Backtrace;
 use complexpr::{eval::Environment, interpreter::interpret, value::Value, stdlib};

src/eval.rs

@@ -114,16 +114,17 @@ pub fn eval_stmt(stmt: &Stmt, env: EnvRef) -> Result<(), Unwind> {
                 return eval_stmt(ec, env)
             }
         },
-        Stmt::For { var, expr, stmt } => {
+        Stmt::For { var, expr, stmt, iter_pos } => {
             let name = unwrap_ident_token(var);
             let iter = eval_expr(expr, env.clone())?;
             env.borrow_mut().declare(name.clone(), Value::Nil);
-            let iterator = iter.iter();
+            let iterator = iter.iter(iter_pos);
             if let Err(e) = iterator {
                 return Err(RuntimeError::new(e, var.pos.clone()).into())
             }
             if let Ok(i) = iterator {
                 for v in i {
+                    let v = v?;
                     let env = env.clone();
                     env.borrow_mut().set(name.clone(), v).expect("unreachable");
                     match eval_stmt(stmt, env) {
@@ -159,11 +160,11 @@ pub fn eval_stmt(stmt: &Stmt, env: EnvRef) -> Result<(), Unwind> {
             };
             env.borrow_mut().declare(name, Value::Func(func));
         },
-        Stmt::Break { tok } => return Err(Unwind::Break { pos: tok.pos.clone() }),
-        Stmt::Continue { tok } => return Err(Unwind::Continue { pos: tok.pos.clone() }),
-        Stmt::Return { tok, expr } => {
+        Stmt::Break { pos } => return Err(Unwind::Break { pos: pos.clone() }),
+        Stmt::Continue { pos } => return Err(Unwind::Continue { pos: pos.clone() }),
+        Stmt::Return { pos, expr } => {
             let value = eval_expr(expr, env)?;
-            return Err(Unwind::Return { pos: tok.pos.clone(), value })
+            return Err(Unwind::Return { pos: pos.clone(), value })
         }
     }
     Ok(())

src/expr.rs

@@ -8,11 +8,11 @@ pub enum Stmt {
     Let { lhs: Token, rhs: Option<Expr> },
     Block { stmts: Vec<Stmt> },
     If { if_clauses: Vec<(Expr, Stmt)>, else_clause: Option<Box<Stmt>> },
-    For { var: Token, expr: Expr, stmt: Box<Stmt> },
+    For { var: Token, expr: Expr, stmt: Box<Stmt>, iter_pos: Position },
     While { expr: Expr, stmt: Box<Stmt> },
-    Break { tok: Token },
-    Continue { tok: Token },
-    Return { tok: Token, expr: Expr },
+    Break { pos: Position },
+    Continue { pos: Position },
+    Return { pos: Position, expr: Expr },
     Fn { name: Token, args: Vec<Token>, body: Box<Stmt> },
 }
 
@@ -22,7 +22,13 @@ impl fmt::Debug for Stmt {
             Self::Expr { expr } => write!(f, "{:?}", expr),
             Self::Let { lhs, rhs } => write!(f, "(let {:?} = {:?})", lhs, rhs),
             Self::Block { stmts } => write!(f, "(block {:?})", stmts),
-            _ => todo!()
+            Self::If { if_clauses, else_clause } => write!(f, "(if {:?} else {:?})", if_clauses, else_clause),
+            Self::For { var, expr, stmt, .. } => write!(f, "(for {:?} : {:?} do {:?})", var, expr, stmt),
+            Self::While { expr, stmt } => write!(f, "(while {:?} do {:?})", expr, stmt),
+            Self::Break { .. } => write!(f, "(break)"),
+            Self::Continue { .. } => write!(f, "(continue)"),
+            Self::Return { expr, .. } => write!(f, "(return {:?})", expr),
+            Self::Fn { name, args, body } => write!(f, "(fn {:?} {:?} {:?})", name, args, body),
         }
     }
 }

src/lexer.rs

@@ -43,6 +43,14 @@ impl Lexer {
         self.current >= self.code.len()
     }
 
+    fn err_on_eof(&self, msg: &str) -> Result<(), ParserError> {
+        if self.at_end() {
+            Err(self.mk_error(msg))
+        } else {
+            Ok(())
+        }
+    }
+
     fn peek(&self) -> char {
         self.code[self.current]
     }
@@ -81,6 +89,53 @@ impl Lexer {
         self.current += 1;
     }
 
+    fn parse_escape(&mut self, eof_msg: &str) -> Result<Option<char>, ParserError> {
+        self.err_on_eof(eof_msg)?;
+        Ok(Some(match self.peek() {
+            '0' => '\0',
+            'n' => '\n',
+            't' => '\t',
+            'r' => '\r',
+            'e' => '\x1b',
+            '\\' => '\\',
+            '"' => '"',
+            '\'' => '\'',
+            '\n' => { return Ok(None) },
+            'x' => {
+                self.advance(false);
+                self.err_on_eof(eof_msg)?;
+                let c1 = self.peek();
+                self.advance(c1 == '\n');
+                self.err_on_eof(eof_msg)?;
+                let c2 = self.peek();
+                let code = format!("{}{}", c1, c2);
+                let code = u32::from_str_radix(&code, 16).map_err(|_| self.mk_error("Invalid hex code in escape"))?;
+                char::from_u32(code).unwrap()
+            },
+            'u' => {
+                self.advance(false);
+                self.err_on_eof(eof_msg)?;
+                if self.peek() != '{' {
+                    return Err(self.mk_error("Expected { to begin unicode escape"))
+                }
+                self.advance(false);
+                self.err_on_eof(eof_msg)?;
+                let mut esc_str = String::new();
+                while self.peek().is_ascii_hexdigit() {
+                    esc_str.push(self.peek());
+                    self.advance(false);
+                    self.err_on_eof(eof_msg)?;
+                }
+                if self.peek() != '}' {
+                    return Err(self.mk_error("Expected } to terminate unicode escape"))
+                }
+                let code = u32::from_str_radix(&esc_str, 16).map_err(|_| self.mk_error("Invalid hex code in escape"))?;
+                char::from_u32(code).ok_or_else(|| self.mk_error("Invalid unicode character"))?
+            },
+            c => return Err(self.mk_error(format!("Unknown escape code \\{}", c)))
+        }))
+    }
+
     pub fn lex(&mut self) -> Result<(), ParserError> {
         while !self.at_end() {
             self.start = self.current;
@@ -172,39 +227,34 @@ impl Lexer {
     }
 
     fn char(&mut self) -> Result<(), ParserError> {
-        if self.at_end() { return Err(self.mk_error("Unexpected EOF in character literal")) }
-        let mut c = self.next();
+        const EOF_MSG: &str = "Unexpected EOF in character literal";
+        self.err_on_eof(EOF_MSG)?;
+        let mut c = self.peek();
         if c == '\'' {
             return Err(self.mk_error("Empty character literal"))
         } else if c == '\\' {
-            if self.at_end() { return Err(self.mk_error("Unexpected EOF in character literal")) }
-            // TODO Escapes
+            self.advance(self.peek() == '\n');
+            if let Some(nc) = self.parse_escape(EOF_MSG)? {
+                c = nc
+            } else {
+                return Err(self.mk_error("Character literal cannot contain escaped newline"));
             }
-        if self.at_end() { return Err(self.mk_error("Unexpected EOF in character literal")) }
+        }
+        self.err_on_eof(EOF_MSG)?;
+        self.advance(self.peek() == '\n');
         self.expect(&['\'']).ok_or_else(|| self.mk_error("Expected ' to terminate character literal"))?;
         self.add_token(TokenType::Char(c), self.collect_literal());
         Ok(())
     }
 
     fn string(&mut self) -> Result<(), ParserError> {
+        const EOF_MSG: &str = "Unexpected EOF in string literal";
         let mut s = String::new();
         while !self.at_end() && self.peek() != '"' {
             if self.peek() == '\\' {
                 self.advance(false);
-                if self.at_end() { 
-                    return Err(self.mk_error("Unexpected EOF in string literal")) 
-                }
-                // TODO more escape codes! \xHH, \u{HH..} or maybe \uhhhh \Uhhhhhhhh
-                match self.peek() {
-                    '0' => s.push('\0'),
-                    'n' => s.push('\n'),
-                    't' => s.push('\t'),
-                    'r' => s.push('\r'),
-                    'e' => s.push('\x1b'),
-                    '\\' => s.push('\\'),
-                    '"' => s.push('"'),
-                    '\n' => (),
-                    c => return Err(self.mk_error(format!("Unknown escape code \\{}", c)))
+                if let Some(c) = self.parse_escape(EOF_MSG)? {
+                    s.push(c);
                 }
                 self.advance(self.peek() == '\n')
             } else {
@@ -212,9 +262,7 @@ impl Lexer {
                 self.advance(self.peek() == '\n');
             }
         }
-        if self.at_end() {
-            return Err(self.mk_error("Unexpected EOF in string literal"))
-        }
+        self.err_on_eof(EOF_MSG)?;
         self.advance(false);
         self.add_token(TokenType::String(Rc::from(s)), self.collect_literal());
         Ok(())

src/lib.rs

@@ -47,14 +47,23 @@ impl RuntimeError {
         }
     }
 
+    pub fn new_incomplete<S>(message: S) -> Self
+    where S: Into<String> {
+        Self { 
+            message: message.into(), 
+            stacktrace: vec![],
+            last_pos: None
+        }
+    }
+
     pub fn exit_fn(mut self, fn_name: Option<Rc<str>>, pos: Position) -> Self {
-        self.stacktrace.push(Stackframe { pos: self.last_pos.unwrap(), fn_name });
+        self.stacktrace.push(Stackframe { pos: self.last_pos.expect("RuntimeError never completed after construction"), fn_name });
         self.last_pos = Some(pos);
         self
     }
 
     pub fn finish(mut self, ctx_name: Option<Rc<str>>) -> Self {
-        self.stacktrace.push(Stackframe { pos: self.last_pos.unwrap(), fn_name: ctx_name });
+        self.stacktrace.push(Stackframe { pos: self.last_pos.expect("RuntimeError never completed after construction"), fn_name: ctx_name });
         self.last_pos = None;
         self
     }
@@ -62,7 +71,7 @@ impl RuntimeError {
 
 impl fmt::Display for ParserError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "Error: {}\n    In {} at {},{}", 
+        write!(f, "Error: {}\n    In {} at {},{}\n", 
             self.message, 
             self.pos.file.as_ref().map(|o| o.as_ref()).unwrap_or("<unknown>"), 
             self.pos.line, 

src/parser.rs

@@ -81,19 +81,19 @@ impl Parser {
             },
             TokenType::Break => {
                 let tok = self.next();
-                self.terminate_stmt(Stmt::Break{ tok })
+                self.terminate_stmt(Stmt::Break{ pos: tok.pos })
             },
             TokenType::Continue => {
                 let tok = self.next();
-                self.terminate_stmt(Stmt::Continue{ tok })
+                self.terminate_stmt(Stmt::Continue{ pos: tok.pos })
             },
             TokenType::Return => {
                 let tok = self.next();
                 let expr = self.assignment()?;
-                self.terminate_stmt(Stmt::Return{ tok, expr })
+                self.terminate_stmt(Stmt::Return{ pos: tok.pos, expr })
             },
             TokenType::Fn => {
-                let tok = self.next();
+                self.next();
                 self.fndef()
             },
             _ => {
@@ -188,15 +188,15 @@ impl Parser {
         let var = self.next();
         if let TokenType::Ident(_) = &var.ty {
             self.err_on_eof()?;
-            let x = self.next();
-            if x.ty != TokenType::Colon {
+            let colon = self.next();
+            if colon.ty != TokenType::Colon {
                 return Err(self.mk_error("Expected colon"))
             }
             self.err_on_eof()?;
             let expr = self.assignment()?;
             self.err_on_eof()?;
             let stmt = self.statement()?;
-            Ok(Stmt::For{ var, expr, stmt: Box::new(stmt) })
+            Ok(Stmt::For{ var, expr, stmt: Box::new(stmt), iter_pos: colon.pos })
         } else {
             Err(self.mk_error("Expected identifier after for"))
         }

src/value.rs

@@ -20,7 +20,19 @@ impl fmt::Debug for BuiltinFunc {
     }
 }
 
-#[derive(Clone, Debug)]
+impl Iterator for BuiltinFunc {
+    type Item = Result<Value, String>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // precondition: function takes zero arguments
+        match (self.func)(vec![]) {
+            Ok(Value::Nil) => None,
+            r => Some(r),
+        }
+    }
+}
+
+#[derive(Clone)]
 pub struct Func {
     pub name: Option<Rc<str>>,
     pub args: Vec<Rc<str>>,
@@ -28,6 +40,31 @@ pub struct Func {
     pub func: Stmt
 }
 
+impl fmt::Debug for Func {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Func")
+            .field("name", &self.name)
+            .field("args", &self.args)
+            .field("func", &self.func)
+            .finish_non_exhaustive()
+    }
+}
+
+impl Iterator for Func {
+    type Item = Result<Value, RuntimeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // precondition: function takes zero arguments
+        let env = Environment::extend(self.env.clone()).wrap();
+        match eval_stmt(&self.func, env) {
+            Ok(_) => None,
+            Err(Unwind::Return{ value: Value::Nil, .. }) => None,
+            Err(Unwind::Return{ value, .. }) => Some(Ok(value)),
+            Err(e) => Some(Err(e.as_error()))
+        }
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct Data {
     pub ty: usize,
@@ -70,12 +107,26 @@ impl Value {
         }
     }
 
-    pub fn iter(&self) -> Result<Box<dyn Iterator<Item=Value> + '_>, String> {
+    pub fn iter<'a>(&'a self, pos: &'a Position) -> Result<Box<dyn Iterator<Item=Result<Value, RuntimeError>> + '_>, String> {
         match self {
             Value::String(s) 
                 => Ok(Box::new(s.chars()
-                    .map(Value::Char))),
-            Value::List(l) => Ok(Box::new(l.borrow().clone().into_iter())),
+                    .map(Value::Char).map(Ok))),
+            Value::List(l) => Ok(Box::new(l.borrow().clone().into_iter().map(Ok))),
+            Value::BuiltinFunc(bf) => {
+                if bf.arg_count == 0 {
+                    Ok(Box::new(bf.clone().map(|e| e.map_err(|e| RuntimeError::new(e, pos.clone())))))
+                } else {
+                    Err("Only zero-argument functions can be used as iterators".into())
+                }
+            },
+            Value::Func(f) => {
+                if f.args.len() == 0 {
+                    Ok(Box::new(f.clone()))
+                } else {
+                    Err("Only zero-argument functions can be used as iterators".into())
+                }
+            },
             v => Err(format!("{:?} is not iterable", v))
         }
     }