feat: Adding debugger to interpreter

cli/tool/debug.go

@@ -12,12 +12,14 @@ package tool
 
 import (
 	"bufio"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"net"
 	"strings"
 	"time"
 
+	"devt.de/krotik/common/stringutil"
 	"devt.de/krotik/ecal/interpreter"
 	"devt.de/krotik/ecal/util"
 )
@@ -67,17 +69,27 @@ func (i *CLIDebugInterpreter) Interpret() error {
 		return nil
 	}
 
-	i.CLIInterpreter.CustomWelcomeMessage = "Running in debug mode - "
-	if *i.RunDebugServer {
-		i.CLIInterpreter.CustomWelcomeMessage += fmt.Sprintf("with debug server on %v - ", *i.DebugServerAddr)
-	}
-	i.CLIInterpreter.CustomWelcomeMessage += "prefix debug commands with ##"
-	i.CustomHelpString = "    @dbg [glob] - List all available debug commands.\n"
-
 	err := i.CreateRuntimeProvider("debug console")
 
 	if err == nil {
 
+		// Set custom messages
+
+		i.CLIInterpreter.CustomWelcomeMessage = "Running in debug mode - "
+		if *i.RunDebugServer {
+			i.CLIInterpreter.CustomWelcomeMessage += fmt.Sprintf("with debug server on %v - ", *i.DebugServerAddr)
+		}
+		i.CLIInterpreter.CustomWelcomeMessage += "prefix debug commands with ##"
+		i.CustomHelpString = "    @dbg [glob] - List all available debug commands.\n"
+
+		// Set debug object on the runtime provider
+
+		i.RuntimeProvider.Debugger = interpreter.NewECALDebugger(i.GlobalVS)
+
+		// Set this object as a custom handler to deal with input.
+
+		i.CustomHandler = i
+
 		if *i.RunDebugServer {
 			debugServer := &debugTelnetServer{*i.DebugServerAddr, "ECALDebugServer: ",
 				nil, true, i, i.RuntimeProvider.Logger}
@@ -97,6 +109,56 @@ func (i *CLIDebugInterpreter) Interpret() error {
 	return err
 }
 
+/*
+CanHandle checks if a given string can be handled by this handler.
+*/
+func (i *CLIDebugInterpreter) CanHandle(s string) bool {
+	return strings.HasPrefix(s, "##") || strings.HasPrefix(s, "@dbg")
+}
+
+/*
+Handle handles a given input string.
+*/
+func (i *CLIDebugInterpreter) Handle(ot OutputTerminal, line string) {
+
+	if strings.HasPrefix(line, "@dbg") {
+
+		args := strings.Fields(line)[1:]
+
+		tabData := []string{"Debug command", "Description"}
+
+		for name, f := range interpreter.DebugCommandsMap {
+			ds := f.DocString()
+
+			if len(args) > 0 && !matchesFulltextSearch(ot, fmt.Sprintf("%v %v", name, ds), args[0]) {
+				continue
+			}
+
+			tabData = fillTableRow(tabData, name, ds)
+		}
+
+		if len(tabData) > 2 {
+			ot.WriteString(stringutil.PrintGraphicStringTable(tabData, 2, 1,
+				stringutil.SingleDoubleLineTable))
+		}
+
+	} else {
+		res, err := i.RuntimeProvider.Debugger.HandleInput(strings.TrimSpace(line[2:]))
+
+		if err == nil {
+			var outBytes []byte
+			outBytes, err = json.MarshalIndent(res, "", "  ")
+			if err == nil {
+				ot.WriteString(fmt.Sprintln(string(outBytes)))
+			}
+		}
+
+		if err != nil {
+			ot.WriteString(fmt.Sprintf("Debugger Error: %v", err.Error()))
+		}
+	}
+}
+
 /*
 debugTelnetServer is a simple telnet server to send and receive debug data.
 */
@@ -148,7 +210,7 @@ HandleConnection handles an incoming connection.
 func (s *debugTelnetServer) HandleConnection(conn net.Conn) {
 	tid := s.interpreter.RuntimeProvider.NewThreadID()
 	inputReader := bufio.NewReader(conn)
-	outputTerminal := interpreter.OutputTerminal(&bufioWriterShim{bufio.NewWriter(conn)})
+	outputTerminal := OutputTerminal(&bufioWriterShim{bufio.NewWriter(conn)})
 
 	line := ""
 

cli/tool/helper.go

@@ -16,7 +16,6 @@ import (
 	"strings"
 
 	"devt.de/krotik/common/stringutil"
-	"devt.de/krotik/ecal/interpreter"
 )
 
 /*
@@ -32,14 +31,25 @@ type CLIInputHandler interface {
 	/*
 	   Handle handles a given input string.
 	*/
-	Handle(ot interpreter.OutputTerminal, args []string)
+	Handle(ot OutputTerminal, input string)
+}
+
+/*
+OutputTerminal is a generic output terminal which can write strings.
+*/
+type OutputTerminal interface {
+
+	/*
+	   WriteString write a string on this terminal.
+	*/
+	WriteString(s string)
 }
 
 /*
 matchesFulltextSearch checks if a given text matches a given glob expression. Returns
 true if an error occurs.
 */
-func matchesFulltextSearch(ot interpreter.OutputTerminal, text string, glob string) bool {
+func matchesFulltextSearch(ot OutputTerminal, text string, glob string) bool {
 	var res bool
 
 	re, err := stringutil.GlobToRegex(glob)

cli/tool/interpret.go

@@ -243,7 +243,7 @@ HandleInput handles input to this interpreter. It parses a given input line
 and outputs on the given output terminal. Requires a thread ID of the executing
 thread - use the RuntimeProvider to generate a unique one.
 */
-func (i *CLIInterpreter) HandleInput(ot interpreter.OutputTerminal, line string, tid uint64) {
+func (i *CLIInterpreter) HandleInput(ot OutputTerminal, line string, tid uint64) {
 
 	// Process the entered line
 
@@ -270,7 +270,7 @@ func (i *CLIInterpreter) HandleInput(ot interpreter.OutputTerminal, line string,
 		i.displayPackage(ot, strings.Split(line, " ")[1:])
 
 	} else if i.CustomHandler != nil && i.CustomHandler.CanHandle(line) {
-		i.CustomHandler.Handle(ot, strings.Split(line, " ")[1:])
+		i.CustomHandler.Handle(ot, line)
 
 	} else {
 		var ierr error
@@ -296,7 +296,7 @@ func (i *CLIInterpreter) HandleInput(ot interpreter.OutputTerminal, line string,
 /*
 displaySymbols lists all available inbuild functions and available stdlib packages of ECAL.
 */
-func (i *CLIInterpreter) displaySymbols(ot interpreter.OutputTerminal, args []string) {
+func (i *CLIInterpreter) displaySymbols(ot OutputTerminal, args []string) {
 
 	tabData := []string{"Inbuild function", "Description"}
 
@@ -338,7 +338,7 @@ func (i *CLIInterpreter) displaySymbols(ot interpreter.OutputTerminal, args []st
 /*
 displayPackage list all available constants and functions of a stdlib package.
 */
-func (i *CLIInterpreter) displayPackage(ot interpreter.OutputTerminal, args []string) {
+func (i *CLIInterpreter) displayPackage(ot OutputTerminal, args []string) {
 
 	_, constSymbols, funcSymbols := stdlib.GetStdlibSymbols()
 

ecal.md

@@ -29,7 +29,7 @@ foobar.doSomething()
 
 Event Sinks
 --
-Event sinks are the core constructs of ECAL which provide concurrency and the means to respond to events of an external system. Sinks provide ECAL with an interface to an [event condition action engine](engine.md) which coordinates the parallel execution of code. Sinks cannot be scoped into modules or objects and are usually declared at the top level. They have the following form:
+Event sinks are the core constructs of ECAL which provide concurrency and the means to respond to events of an external system. Sinks provide ECAL with an interface to an [event condition action engine](engine.md) which coordinates the parallel execution of code. Sinks cannot be scoped into modules or objects and are usually declared at the top level. Sinks must not write to top level variables. They have the following form:
 ```
 sink mysink
     kindmatch [ "foo.bar.*" ],
@@ -357,7 +357,7 @@ Build-in Functions
 --
 ECAL has a number of function which are build-in that are always available:
 
-#### `raise([error type], [error detail], [data) : error`
+#### `raise([error type], [error detail], [data]) : error`
 Raise returns a runtime error. Outside of sinks this will stop the code execution
 if the error is not handled by try / except. Inside a sink only the specific sink
 will fail.

engine/pool/threadpool.go

@@ -159,7 +159,7 @@ NewThreadPoolWithQueue creates a new thread pool with a specific task queue.
 */
 func NewThreadPoolWithQueue(q TaskQueue) *ThreadPool {
 	return &ThreadPool{q, &sync.Mutex{},
-		0, &sync.Mutex{}, make(map[uint64]*ThreadPoolWorker),
+		1, &sync.Mutex{}, make(map[uint64]*ThreadPoolWorker),
 		make(map[uint64]*ThreadPoolWorker), &sync.Mutex{},
 		0, sync.NewCond(&sync.Mutex{}), &sync.Mutex{},
 		math.MaxInt32, func() {}, false, 0, func() {}, false}

engine/processor.go

@@ -347,8 +347,8 @@ func (p *eventProcessor) AddEvent(event *Event, eventMonitor Monitor) (Monitor,
 
 	// Check that the thread pool is running
 
-	if p.pool.Status() == pool.StatusStopped {
-		return nil, fmt.Errorf("Cannot add event if the processor is not running")
+	if s := p.pool.Status(); s == pool.StatusStopped || s == pool.StatusStopping {
+		return nil, fmt.Errorf("Cannot add event if the processor is stopping or not running")
 	}
 
 	EventTracer.record(event, "eventProcessor.AddEvent", "Event added to the processor")

interpreter/debug.go

@@ -13,20 +13,490 @@ Package interpreter contains the ECAL interpreter.
 */
 package interpreter
 
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/ecal/parser"
+	"devt.de/krotik/ecal/scope"
+	"devt.de/krotik/ecal/util"
+)
+
+/*
+ecalDebugger is the inbuild default debugger.
+*/
+type ecalDebugger struct {
+	breakPoints         map[string]bool                // Break points (active or not)
+	interrogationStates map[uint64]*interrogationState // Collection of threads which are interrogated
+	callStacks          map[uint64][]*parser.ASTNode   // Call stacks of threads
+	sources             map[string]bool                // All known sources
+	breakOnStart        bool                           // Flag to stop at the start of the next execution
+	globalScope         parser.Scope                   // Global variable scope which can be used to transfer data
+	lock                *sync.RWMutex                  // Lock for this debugger
+
+}
+
+/*
+interrogationState contains state information of a thread interrogation.
+*/
+type interrogationState struct {
+	cond         *sync.Cond        // Condition on which the thread is waiting when suspended
+	running      bool              // Flag if the thread is running or waiting
+	cmd          interrogationCmd  // Next interrogation command for the thread
+	stepOutStack []*parser.ASTNode // Target stack when doing a step out
+	node         *parser.ASTNode   // Node on which the thread was last stopped
+	vs           parser.Scope      // Variable scope of the thread when it was last stopped
+}
+
+/*
+interrogationCmd represents a command for a thread interrogation.
+*/
+type interrogationCmd int
+
+/*
+Interrogation commands
+*/
+const (
+	Stop     interrogationCmd = iota // Stop the execution (default)
+	StepIn                           // Step into the next function
+	StepOut                          // Step out of the current function
+	StepOver                         // Step over the next function
+	Resume                           // Resume execution - do not break again on the same line
+)
+
+/*
+newInterrogationState creates a new interrogation state.
+*/
+func newInterrogationState(node *parser.ASTNode, vs parser.Scope) *interrogationState {
+	return &interrogationState{
+		sync.NewCond(&sync.Mutex{}),
+		false,
+		Stop,
+		nil,
+		node,
+		vs,
+	}
+}
+
+/*
+NewDebugger returns a new debugger object.
+*/
+func NewECALDebugger(globalVS parser.Scope) util.ECALDebugger {
+	return &ecalDebugger{
+		breakPoints:         make(map[string]bool),
+		interrogationStates: make(map[uint64]*interrogationState),
+		callStacks:          make(map[uint64][]*parser.ASTNode),
+		sources:             make(map[string]bool),
+		breakOnStart:        false,
+		globalScope:         globalVS,
+		lock:                &sync.RWMutex{},
+	}
+}
+
+/*
+HandleInput handles a given debug instruction from a console.
+*/
+func (ed *ecalDebugger) HandleInput(input string) (interface{}, error) {
+	var res interface{}
+	var err error
+
+	args := strings.Fields(input)
+
+	if cmd, ok := DebugCommandsMap[args[0]]; ok {
+		if len(args) > 1 {
+			res, err = cmd.Run(ed, args[1:])
+		} else {
+			res, err = cmd.Run(ed, nil)
+		}
+	} else {
+		err = fmt.Errorf("Unknown command: %v", args[0])
+	}
+
+	return res, err
+}
+
+/*
+Break on the start of the next execution.
+*/
+func (ed *ecalDebugger) BreakOnStart(flag bool) {
+	ed.lock.Lock()
+	defer ed.lock.Unlock()
+	ed.breakOnStart = flag
+}
+
+/*
+VisitState is called for every state during the execution of a program.
+*/
+func (ed *ecalDebugger) VisitState(node *parser.ASTNode, vs parser.Scope, tid uint64) util.TraceableRuntimeError {
+
+	ed.lock.RLock()
+	_, ok := ed.callStacks[tid]
+	ed.lock.RUnlock()
+
+	if !ok {
+
+		// Make the debugger aware of running threads
+
+		ed.lock.Lock()
+		ed.callStacks[tid] = make([]*parser.ASTNode, 0, 10)
+		ed.lock.Unlock()
+	}
+
+	if node.Token != nil { // Statements are excluded here
+		targetIdentifier := fmt.Sprintf("%v:%v", node.Token.Lsource, node.Token.Lline)
+
+		ed.lock.RLock()
+		is, ok := ed.interrogationStates[tid]
+		_, sourceKnown := ed.sources[node.Token.Lsource]
+		ed.lock.RUnlock()
+
+		if !sourceKnown {
+			ed.RecordSource(node.Token.Lsource)
+		}
+
+		if ok {
+
+			// The thread is being interrogated
+
+			switch is.cmd {
+			case Resume:
+				if is.node.Token.Lline != node.Token.Lline {
+
+					// Remove the resume command once we are on a different line
+
+					ed.lock.Lock()
+					delete(ed.interrogationStates, tid)
+					ed.lock.Unlock()
+
+					return ed.VisitState(node, vs, tid)
+				}
+			case Stop, StepIn, StepOver:
+
+				if is.node.Token.Lline != node.Token.Lline || is.cmd == Stop {
+					is.node = node
+					is.vs = vs
+					is.running = false
+
+					is.cond.L.Lock()
+					is.cond.Wait()
+					is.cond.L.Unlock()
+				}
+			}
+
+		} else if active, ok := ed.breakPoints[targetIdentifier]; (ok && active) || ed.breakOnStart {
+
+			// A globally defined breakpoint has been hit - note the position
+			// in the thread specific map and wait
+
+			is := newInterrogationState(node, vs)
+
+			ed.lock.Lock()
+			ed.breakOnStart = false
+			ed.interrogationStates[tid] = is
+			ed.lock.Unlock()
+
+			is.cond.L.Lock()
+			is.cond.Wait()
+			is.cond.L.Unlock()
+		}
+	}
+
+	return nil
+}
+
 /*
-OutputTerminal is a generic output terminal which can write strings.
+VisitStepInState is called before entering a function call.
 */
-type OutputTerminal interface {
+func (ed *ecalDebugger) VisitStepInState(node *parser.ASTNode, vs parser.Scope, tid uint64) util.TraceableRuntimeError {
+	ed.lock.Lock()
+	defer ed.lock.Unlock()
+
+	var err util.TraceableRuntimeError
+
+	threadCallStack := ed.callStacks[tid]
+
+	is, ok := ed.interrogationStates[tid]
+
+	if ok {
+
+		if is.cmd == Stop {
+
+			// Special case a parameter of a function was resolved by another
+			// function call - the debugger should stop before entering
+
+			ed.lock.Unlock()
+			err = ed.VisitState(node, vs, tid)
+			ed.lock.Lock()
+		}
+
+		if err == nil {
+			// The thread is being interrogated
+
+			switch is.cmd {
+			case StepIn:
+				is.cmd = Stop
+			case StepOver:
+				is.cmd = StepOut
+				is.stepOutStack = ed.callStacks[tid]
+			}
+		}
+	}
+
+	ed.callStacks[tid] = append(threadCallStack, node)
+
+	return err
+}
+
+/*
+VisitStepOutState is called after returning from a function call.
+*/
+func (ed *ecalDebugger) VisitStepOutState(node *parser.ASTNode, vs parser.Scope, tid uint64) util.TraceableRuntimeError {
+	ed.lock.Lock()
+	defer ed.lock.Unlock()
+
+	threadCallStack := ed.callStacks[tid]
+	lastIndex := len(threadCallStack) - 1
+
+	ok, cerr := threadCallStack[lastIndex].Equals(node, false) // Sanity check step in node must be the same as step out node
+	errorutil.AssertTrue(ok,
+		fmt.Sprintf("Unexpected callstack when stepping out - callstack: %v - funccall: %v - comparison error: %v",
+			threadCallStack, node, cerr))
+
+	ed.callStacks[tid] = threadCallStack[:lastIndex] // Remove the last item
+
+	is, ok := ed.interrogationStates[tid]
+
+	if ok {
+
+		// The thread is being interrogated
+
+		switch is.cmd {
+		case StepOver, StepOut:
+
+			if len(ed.callStacks[tid]) == len(is.stepOutStack) {
+				is.cmd = Stop
+			}
+		}
+	}
+
+	return nil
+}
+
+/*
+RecordSource records a code source.
+*/
+func (ed *ecalDebugger) RecordSource(source string) {
+	ed.lock.Lock()
+	defer ed.lock.Unlock()
+	ed.sources[source] = true
+}
+
+/*
+SetBreakPoint sets a break point.
+*/
+func (ed *ecalDebugger) SetBreakPoint(source string, line int) {
+	ed.lock.Lock()
+	defer ed.lock.Unlock()
+	ed.breakPoints[fmt.Sprintf("%v:%v", source, line)] = true
+}
+
+/*
+DisableBreakPoint disables a break point but keeps the code reference.
+*/
+func (ed *ecalDebugger) DisableBreakPoint(source string, line int) {
+	ed.lock.Lock()
+	defer ed.lock.Unlock()
+	ed.breakPoints[fmt.Sprintf("%v:%v", source, line)] = false
+}
+
+/*
+RemoveBreakPoint removes a break point.
+*/
+func (ed *ecalDebugger) RemoveBreakPoint(source string, line int) {
+	ed.lock.Lock()
+	defer ed.lock.Unlock()
+	delete(ed.breakPoints, fmt.Sprintf("%v:%v", source, line))
+}
+
+/*
+ExtractValue copies a value from a suspended thread into the
+global variable scope.
+*/
+func (ed *ecalDebugger) ExtractValue(threadId uint64, varName string, destVarName string) error {
+	if ed.globalScope == nil {
+		return fmt.Errorf("Cannot access global scope")
+	}
+
+	err := fmt.Errorf("Cannot find suspended thread %v", threadId)
+
+	ed.lock.Lock()
+	defer ed.lock.Unlock()
+
+	is, ok := ed.interrogationStates[threadId]
+
+	if ok && !is.running {
+		var val interface{}
+		var ok bool
+
+		if val, ok, err = is.vs.GetValue(varName); ok {
+			err = ed.globalScope.SetValue(destVarName, val)
+		} else if err == nil {
+			err = fmt.Errorf("No such value %v", varName)
+		}
+	}
+
+	return err
+}
+
+/*
+InjectValue copies a value from an expression (using the global variable scope) into
+a suspended thread.
+*/
+func (ed *ecalDebugger) InjectValue(threadId uint64, varName string, expression string) error {
+	if ed.globalScope == nil {
+		return fmt.Errorf("Cannot access global scope")
+	}
+
+	err := fmt.Errorf("Cannot find suspended thread %v", threadId)
+
+	ed.lock.Lock()
+	defer ed.lock.Unlock()
+
+	is, ok := ed.interrogationStates[threadId]
+
+	if ok && !is.running {
+		var ast *parser.ASTNode
+		var val interface{}
+
+		// Eval expression
+
+		ast, err = parser.ParseWithRuntime("InjectValueExpression", expression,
+			NewECALRuntimeProvider("InjectValueExpression2", nil, nil))
+
+		if err == nil {
+			if err = ast.Runtime.Validate(); err == nil {
+
+				ivs := scope.NewScopeWithParent("InjectValueExpressionScope", ed.globalScope)
+				val, err = ast.Runtime.Eval(ivs, make(map[string]interface{}), 999)
+
+				if err == nil {
+					err = is.vs.SetValue(varName, val)
+				}
+			}
+		}
+	}
+
+	return err
+}
+
+/*
+Continue will continue a suspended thread.
+*/
+func (ed *ecalDebugger) Continue(threadId uint64, contType util.ContType) {
+	ed.lock.RLock()
+	defer ed.lock.RUnlock()
+
+	if is, ok := ed.interrogationStates[threadId]; ok && !is.running {
+
+		switch contType {
+		case util.Resume:
+			is.cmd = Resume
+		case util.StepIn:
+			is.cmd = StepIn
+		case util.StepOver:
+			is.cmd = StepOver
+		case util.StepOut:
+			is.cmd = StepOut
+			stack := ed.callStacks[threadId]
+			is.stepOutStack = stack[:len(stack)-1]
+		}
+
+		is.running = true
+
+		is.cond.L.Lock()
+		is.cond.Broadcast()
+		is.cond.L.Unlock()
+	}
+}
+
+/*
+Status returns the current status of the debugger.
+*/
+func (ed *ecalDebugger) Status() interface{} {
+	ed.lock.RLock()
+	defer ed.lock.RUnlock()
+
+	var sources []string
+
+	threadStates := make(map[string]map[string]interface{})
+
+	res := map[string]interface{}{
+		"breakpoints":  ed.breakPoints,
+		"breakonstart": ed.breakOnStart,
+		"threads":      threadStates,
+	}
+
+	for k := range ed.sources {
+		sources = append(sources, k)
+	}
+	res["sources"] = sources
+
+	for k, v := range ed.callStacks {
+		s := map[string]interface{}{
+			"callStack": ed.prettyPrintCallStack(v),
+		}
+
+		if is, ok := ed.interrogationStates[k]; ok {
+			s["threadRunning"] = is.running
+		}
+
+		threadStates[fmt.Sprint(k)] = s
+	}
+
+	return res
+}
+
+/*
+Describe decribes a thread currently observed by the debugger.
+*/
+func (ed *ecalDebugger) Describe(threadId uint64) interface{} {
+	ed.lock.RLock()
+	defer ed.lock.RUnlock()
+
+	var res map[string]interface{}
+
+	threadCallStack, ok1 := ed.callStacks[threadId]
+
+	if is, ok2 := ed.interrogationStates[threadId]; ok1 && ok2 {
+
+		res = map[string]interface{}{
+			"threadRunning": is.running,
+			"callStack":     ed.prettyPrintCallStack(threadCallStack),
+		}
+
+		if !is.running {
+
+			codeString, _ := parser.PrettyPrint(is.node)
+			res["code"] = codeString
+			res["node"] = is.node.ToJSONObject()
+			res["vs"] = is.vs.ToJSONObject()
+		}
+	}
 
-	/*
-	   WriteString write a string on this terminal.
-	*/
-	WriteString(s string)
+	return res
 }
 
 /*
-Debugger is a debugging object which can be used to inspect and modify a running
-ECAL environment.
+Describe decribes a thread currently observed by the debugger.
 */
-type Debugger interface {
+func (ed *ecalDebugger) prettyPrintCallStack(threadCallStack []*parser.ASTNode) []string {
+	cs := []string{}
+	for _, s := range threadCallStack {
+		pp, _ := parser.PrettyPrint(s)
+		cs = append(cs, fmt.Sprintf("%v (%v:%v)",
+			pp, s.Token.Lsource, s.Token.Lline))
+	}
+	return cs
 }

interpreter/debug_cmd.go

@@ -0,0 +1,402 @@
+/*
+ * ECAL
+ *
+ * Copyright 2020 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+/*
+Package interpreter contains the ECAL interpreter.
+*/
+package interpreter
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"devt.de/krotik/ecal/parser"
+	"devt.de/krotik/ecal/util"
+)
+
+/*
+InbuildDebugCommandsMap contains the mapping of inbuild debug commands.
+*/
+var DebugCommandsMap = map[string]util.DebugCommand{
+	"breakonstart": &breakOnStartCommand{&inbuildDebugCommand{}},
+	"break":        &setBreakpointCommand{&inbuildDebugCommand{}},
+	"rmbreak":      &rmBreakpointCommand{&inbuildDebugCommand{}},
+	"disablebreak": &disableBreakpointCommand{&inbuildDebugCommand{}},
+	"cont":         &contCommand{&inbuildDebugCommand{}},
+	"describe":     &describeCommand{&inbuildDebugCommand{}},
+	"status":       &statusCommand{&inbuildDebugCommand{}},
+	"extract":      &extractCommand{&inbuildDebugCommand{}},
+	"inject":       &injectCommand{&inbuildDebugCommand{}},
+}
+
+/*
+inbuildDebugCommand is the base structure for inbuild debug commands providing some
+utility functions.
+*/
+type inbuildDebugCommand struct {
+}
+
+/*
+AssertNumParam converts a parameter into a number.
+*/
+func (ibf *inbuildDebugCommand) AssertNumParam(index int, val string) (uint64, error) {
+	if resNum, err := strconv.ParseInt(fmt.Sprint(val), 10, 0); err == nil {
+		return uint64(resNum), nil
+	}
+	return 0, fmt.Errorf("Parameter %v should be a number", index)
+}
+
+// break
+// =====
+
+/*
+setBreakpointCommand sets a breakpoint
+*/
+type setBreakpointCommand struct {
+	*inbuildDebugCommand
+}
+
+/*
+Execute the debug command and return its result. It must be possible to
+convert the output data into a JSON string.
+*/
+func (c *setBreakpointCommand) Run(debugger util.ECALDebugger, args []string) (interface{}, error) {
+	if len(args) == 0 {
+		return nil, fmt.Errorf("Need a break target (<source>:<line>) as first parameter")
+	}
+
+	targetSplit := strings.Split(args[0], ":")
+
+	if len(targetSplit) > 1 {
+		if line, err := strconv.Atoi(targetSplit[1]); err == nil {
+
+			debugger.SetBreakPoint(targetSplit[0], line)
+
+			return nil, nil
+		}
+	}
+
+	return nil, fmt.Errorf("Invalid break target - should be <source>:<line>")
+}
+
+/*
+DocString returns a descriptive text about this command.
+*/
+func (c *setBreakpointCommand) DocString() string {
+	return "Set a breakpoint specifying <source>:<line>"
+}
+
+// breakOnStartCommand
+// ===================
+
+/*
+breakOnStartCommand breaks on the start of the next execution.
+*/
+type breakOnStartCommand struct {
+	*inbuildDebugCommand
+}
+
+/*
+Execute the debug command and return its result. It must be possible to
+convert the output data into a JSON string.
+*/
+func (c *breakOnStartCommand) Run(debugger util.ECALDebugger, args []string) (interface{}, error) {
+	b := true
+	if len(args) > 0 {
+		b, _ = strconv.ParseBool(args[0])
+	}
+	debugger.BreakOnStart(b)
+	return nil, nil
+}
+
+/*
+DocString returns a descriptive text about this command.
+*/
+func (c *breakOnStartCommand) DocString() string {
+	return "Break on the start of the next execution."
+}
+
+// rmbreak
+// =======
+
+/*
+rmBreakpointCommand removes a breakpoint
+*/
+type rmBreakpointCommand struct {
+	*inbuildDebugCommand
+}
+
+/*
+Execute the debug command and return its result. It must be possible to
+convert the output data into a JSON string.
+*/
+func (c *rmBreakpointCommand) Run(debugger util.ECALDebugger, args []string) (interface{}, error) {
+	if len(args) == 0 {
+		return nil, fmt.Errorf("Need a break target (<source>:<line>) as first parameter")
+	}
+
+	targetSplit := strings.Split(args[0], ":")
+
+	if len(targetSplit) > 1 {
+
+		if line, err := strconv.Atoi(targetSplit[1]); err == nil {
+
+			debugger.RemoveBreakPoint(targetSplit[0], line)
+
+			return nil, nil
+		}
+	}
+
+	return nil, fmt.Errorf("Invalid break target - should be <source>:<line>")
+}
+
+/*
+DocString returns a descriptive text about this command.
+*/
+func (c *rmBreakpointCommand) DocString() string {
+	return "Remove a breakpoint specifying <source>:<line>"
+}
+
+// disablebreak
+// ============
+
+/*
+disableBreakpointCommand temporarily disables a breakpoint
+*/
+type disableBreakpointCommand struct {
+	*inbuildDebugCommand
+}
+
+/*
+Execute the debug command and return its result. It must be possible to
+convert the output data into a JSON string.
+*/
+func (c *disableBreakpointCommand) Run(debugger util.ECALDebugger, args []string) (interface{}, error) {
+	if len(args) == 0 {
+		return nil, fmt.Errorf("Need a break target (<source>:<line>) as first parameter")
+	}
+
+	targetSplit := strings.Split(args[0], ":")
+
+	if len(targetSplit) > 1 {
+
+		if line, err := strconv.Atoi(targetSplit[1]); err == nil {
+
+			debugger.DisableBreakPoint(targetSplit[0], line)
+
+			return nil, nil
+		}
+	}
+
+	return nil, fmt.Errorf("Invalid break target - should be <source>:<line>")
+}
+
+/*
+DocString returns a descriptive text about this command.
+*/
+func (c *disableBreakpointCommand) DocString() string {
+	return "Temporarily disable a breakpoint specifying <source>:<line>"
+}
+
+// cont
+// ====
+
+/*
+contCommand continues a suspended thread
+*/
+type contCommand struct {
+	*inbuildDebugCommand
+}
+
+/*
+Execute the debug command and return its result. It must be possible to
+convert the output data into a JSON string.
+*/
+func (c *contCommand) Run(debugger util.ECALDebugger, args []string) (interface{}, error) {
+	var cmd util.ContType
+
+	if len(args) != 2 {
+		return nil, fmt.Errorf("Need a thread ID and a command Resume, StepIn, StepOver or StepOut")
+	}
+
+	threadId, err := c.AssertNumParam(1, args[0])
+
+	if err == nil {
+		cmdString := strings.ToLower(args[1])
+		switch cmdString {
+		case "resume":
+			cmd = util.Resume
+		case "stepin":
+			cmd = util.StepIn
+		case "stepover":
+			cmd = util.StepOver
+		case "stepout":
+			cmd = util.StepOut
+		default:
+			return nil, fmt.Errorf("Invalid command %v - must be resume, stepin, stepover or stepout", cmdString)
+		}
+
+		debugger.Continue(threadId, cmd)
+	}
+
+	return nil, err
+}
+
+/*
+DocString returns a descriptive text about this command.
+*/
+func (c *contCommand) DocString() string {
+	return "Continues a suspended thread. Specify <threadId> <Resume | StepIn | StepOver | StepOut>"
+}
+
+// describe
+// ========
+
+/*
+describeCommand describes a suspended thread
+*/
+type describeCommand struct {
+	*inbuildDebugCommand
+}
+
+/*
+Execute the debug command and return its result. It must be possible to
+convert the output data into a JSON string.
+*/
+func (c *describeCommand) Run(debugger util.ECALDebugger, args []string) (interface{}, error) {
+	var res interface{}
+
+	if len(args) != 1 {
+		return nil, fmt.Errorf("Need a thread ID")
+	}
+
+	threadId, err := c.AssertNumParam(1, args[0])
+
+	if err == nil {
+
+		res = debugger.Describe(threadId)
+	}
+
+	return res, err
+}
+
+/*
+DocString returns a descriptive text about this command.
+*/
+func (c *describeCommand) DocString() string {
+	return "Describes a suspended thread."
+}
+
+// status
+// ======
+
+/*
+statusCommand shows breakpoints and suspended threads
+*/
+type statusCommand struct {
+	*inbuildDebugCommand
+}
+
+/*
+Execute the debug command and return its result. It must be possible to
+convert the output data into a JSON string.
+*/
+func (c *statusCommand) Run(debugger util.ECALDebugger, args []string) (interface{}, error) {
+	return debugger.Status(), nil
+}
+
+/*
+DocString returns a descriptive text about this command.
+*/
+func (c *statusCommand) DocString() string {
+	return "Shows breakpoints and suspended threads."
+}
+
+// extract
+// =======
+
+/*
+extractCommand copies a value from a suspended thread into the
+global variable scope
+*/
+type extractCommand struct {
+	*inbuildDebugCommand
+}
+
+/*
+Execute the debug command and return its result. It must be possible to
+convert the output data into a JSON string.
+*/
+func (c *extractCommand) Run(debugger util.ECALDebugger, args []string) (interface{}, error) {
+	if len(args) != 3 {
+		return nil, fmt.Errorf("Need a thread ID, a variable name and a destination variable name")
+	}
+
+	threadId, err := c.AssertNumParam(1, args[0])
+
+	if err == nil {
+		if !parser.NamePattern.MatchString(args[1]) || !parser.NamePattern.MatchString(args[2]) {
+			err = fmt.Errorf("Variable names may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character")
+		}
+
+		if err == nil {
+			err = debugger.ExtractValue(threadId, args[1], args[2])
+		}
+	}
+
+	return nil, err
+}
+
+/*
+DocString returns a descriptive text about this command.
+*/
+func (c *extractCommand) DocString() string {
+	return "Copies a value from a suspended thread into the global variable scope."
+}
+
+// inject
+// =======
+
+/*
+injectCommand copies a value from the global variable scope into
+a suspended thread
+*/
+type injectCommand struct {
+	*inbuildDebugCommand
+}
+
+/*
+Execute the debug command and return its result. It must be possible to
+convert the output data into a JSON string.
+*/
+func (c *injectCommand) Run(debugger util.ECALDebugger, args []string) (interface{}, error) {
+	if len(args) < 3 {
+		return nil, fmt.Errorf("Need a thread ID, a variable name and an expression")
+	}
+
+	threadId, err := c.AssertNumParam(1, args[0])
+
+	if err == nil {
+		varName := args[1]
+		expression := strings.Join(args[2:], " ")
+
+		err = debugger.InjectValue(threadId, varName, expression)
+	}
+
+	return nil, err
+}
+
+/*
+DocString returns a descriptive text about this command.
+*/
+func (c *injectCommand) DocString() string {
+	return "Copies a value from the global variable scope into a suspended thread."
+}

interpreter/func_provider.go

@@ -832,10 +832,14 @@ func (ct *setCronTrigger) Run(instanceID string, vs parser.Scope, is map[string]
 					"tick":      float64(tick),
 				})
 				monitor := proc.NewRootMonitor(nil, nil)
+
 				_, err := proc.AddEvent(event, monitor)
-				errorutil.AssertTrue(err == nil,
-					fmt.Sprintf("Could not add cron event for trigger %v %v %v: %v",
-						cronspec, eventname, eventkind, err))
+
+				if status := proc.Status(); status != "Stopped" && status != "Stopping" {
+					errorutil.AssertTrue(err == nil,
+						fmt.Sprintf("Could not add cron event for trigger %v %v %v: %v",
+							cronspec, eventname, eventkind, err))
+				}
 			})
 		}
 	}
@@ -904,7 +908,7 @@ func (pt *setPulseTrigger) Run(instanceID string, vs parser.Scope, is map[string
 					monitor := proc.NewRootMonitor(nil, nil)
 					_, err := proc.AddEvent(event, monitor)
 
-					if proc.Stopped() {
+					if status := proc.Status(); status == "Stopped" || status == "Stopping" {
 						break
 					}
 

interpreter/main_test.go

@@ -14,6 +14,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"sync"
 	"testing"
 
 	"devt.de/krotik/common/datautil"
@@ -50,6 +51,11 @@ func TestMain(m *testing.M) {
 var usedNodes = map[string]bool{
 	parser.NodeEOF: true,
 }
+var usedNodesLock = &sync.Mutex{}
+
+// Debuggger to be used
+//
+var testDebugger util.ECALDebugger
 
 // Last used logger
 //
@@ -70,7 +76,19 @@ func UnitTestEvalAndAST(input string, vs parser.Scope, expectedAST string) (inte
 	return UnitTestEvalAndASTAndImport(input, vs, expectedAST, nil)
 }
 
-func UnitTestEvalAndASTAndImport(input string, vs parser.Scope, expectedAST string, importLocator util.ECALImportLocator) (interface{}, error) {
+func UnitTestEvalWithRuntimeProvider(input string, vs parser.Scope,
+	erp *ECALRuntimeProvider) (interface{}, error) {
+	return UnitTestEvalAndASTAndImportAndRuntimeProvider(input, vs, "", nil, erp)
+}
+
+func UnitTestEvalAndASTAndImport(input string, vs parser.Scope, expectedAST string,
+	importLocator util.ECALImportLocator) (interface{}, error) {
+	return UnitTestEvalAndASTAndImportAndRuntimeProvider(input, vs, expectedAST, importLocator, nil)
+}
+
+func UnitTestEvalAndASTAndImportAndRuntimeProvider(input string, vs parser.Scope, expectedAST string,
+	importLocator util.ECALImportLocator, erp *ECALRuntimeProvider) (interface{}, error) {
+
 	var traverseAST func(n *parser.ASTNode)
 
 	traverseAST = func(n *parser.ASTNode) {
@@ -78,7 +96,9 @@ func UnitTestEvalAndASTAndImport(input string, vs parser.Scope, expectedAST stri
 			panic(fmt.Sprintf("Node found with empty string name: %s", n))
 		}
 
+		usedNodesLock.Lock()
 		usedNodes[n.Name] = true
+		usedNodesLock.Unlock()
 		for _, cn := range n.Children {
 			traverseAST(cn)
 		}
@@ -86,7 +106,13 @@ func UnitTestEvalAndASTAndImport(input string, vs parser.Scope, expectedAST stri
 
 	// Parse the input
 
-	erp := NewECALRuntimeProvider("ECALTestRuntime", importLocator, nil)
+	if erp == nil {
+		erp = NewECALRuntimeProvider("ECALTestRuntime", importLocator, nil)
+	}
+
+	// Set debugger
+
+	erp.Debugger = testDebugger
 
 	testlogger = erp.Logger.(*util.MemoryLogger)
 
@@ -124,7 +150,7 @@ func UnitTestEvalAndASTAndImport(input string, vs parser.Scope, expectedAST stri
 		vs = scope.NewScope(scope.GlobalScope)
 	}
 
-	return ast.Runtime.Eval(vs, make(map[string]interface{}), 0)
+	return ast.Runtime.Eval(vs, make(map[string]interface{}), erp.NewThreadID())
 }
 
 /*

interpreter/provider.go

@@ -140,7 +140,7 @@ type ECALRuntimeProvider struct {
 	Logger        util.Logger            // Logger object for log messages
 	Processor     engine.Processor       // Processor of the ECA engine
 	Cron          *timeutil.Cron         // Cron object for scheduled execution
-	Debugger      Debugger               // Optional: Debugger object
+	Debugger      util.ECALDebugger      // Optional: ECAL Debugger object
 }
 
 /*

interpreter/rt_general.go

@@ -29,6 +29,7 @@ type baseRuntime struct {
 	instanceID string               // Unique identifier (should be used when instance state is stored)
 	erp        *ECALRuntimeProvider // Runtime provider
 	node       *parser.ASTNode      // AST node which this runtime component is servicing
+	validated  bool
 }
 
 var instanceCounter uint64 // Global instance counter to create unique identifiers for every runtime component instance
@@ -37,6 +38,7 @@ var instanceCounter uint64 // Global instance counter to create unique identifie
 Validate this node and all its child nodes.
 */
 func (rt *baseRuntime) Validate() error {
+	rt.validated = true
 
 	// Validate all children
 
@@ -53,7 +55,15 @@ func (rt *baseRuntime) Validate() error {
 Eval evaluate this runtime component.
 */
 func (rt *baseRuntime) Eval(vs parser.Scope, is map[string]interface{}, tid uint64) (interface{}, error) {
-	return nil, nil
+	var err error
+
+	errorutil.AssertTrue(rt.validated, "Runtime component has not been validated - please call Validate() before Eval()")
+
+	if rt.erp.Debugger != nil {
+		err = rt.erp.Debugger.VisitState(rt.node, vs, tid)
+	}
+
+	return nil, err
 }
 
 /*
@@ -61,7 +71,7 @@ newBaseRuntime returns a new instance of baseRuntime.
 */
 func newBaseRuntime(erp *ECALRuntimeProvider, node *parser.ASTNode) *baseRuntime {
 	instanceCounter++
-	return &baseRuntime{fmt.Sprint(instanceCounter), erp, node}
+	return &baseRuntime{fmt.Sprint(instanceCounter), erp, node, false}
 }
 
 // Void Runtime

interpreter/rt_general_test.go

@@ -59,6 +59,7 @@ func TestGeneralCases(t *testing.T) {
 	n, _ = parser.Parse("a", "a")
 	void := &voidRuntime{newBaseRuntime(NewECALRuntimeProvider("a", nil, nil), n)}
 	n.Runtime = void
+	void.Validate()
 
 	if res, err := void.Eval(nil, nil, 0); err != nil || res != nil {
 		t.Error("Unexpected result:", res, err)
@@ -103,6 +104,7 @@ statements
 	n.Runtime = imp
 	imp.erp = NewECALRuntimeProvider("ECALTestRuntime", nil, nil)
 	imp.erp.ImportLocator = nil
+	imp.Validate()
 
 	if res, err := imp.Eval(nil, nil, 0); err == nil || err.Error() != "ECAL error in ECALTestRuntime: Runtime error (No import locator was specified) (Line:1 Pos:1)" {
 		t.Error("Unexpected result:", res, err)

interpreter/rt_identifier.go

@@ -181,7 +181,7 @@ func (rt *identifierRuntime) resolveFunction(astring string, vs parser.Scope, is
 						// Convert non-string structures
 
 						for i, a := range args {
-							if _, ok := args[i].(string); !ok {
+							if _, ok := a.(string); !ok {
 								args[i] = stringutil.ConvertToPrettyString(a)
 							}
 						}
@@ -196,11 +196,19 @@ func (rt *identifierRuntime) resolveFunction(astring string, vs parser.Scope, is
 
 					} else {
 
-						// Execute the function and
+						if rt.erp.Debugger != nil {
+							rt.erp.Debugger.VisitStepInState(node, vs, tid)
+						}
+
+						// Execute the function
 
 						result, err = funcObj.Run(rt.instanceID, vs, is, tid, args)
 
-						_, ok1 := err.(*util.RuntimeErrorWithDetail)
+						if rt.erp.Debugger != nil {
+							rt.erp.Debugger.VisitStepOutState(node, vs, tid)
+						}
+
+						_, ok1 := err.(*util.RuntimeError)
 						_, ok2 := err.(*util.RuntimeErrorWithDetail)
 
 						if err != nil && !ok1 && !ok2 {
@@ -216,6 +224,13 @@ func (rt *identifierRuntime) resolveFunction(astring string, vs parser.Scope, is
 
 							err = rerr
 						}
+
+						if tr, ok := err.(util.TraceableRuntimeError); ok {
+
+							// Add tracing information to the error
+
+							tr.AddTrace(rt.node)
+						}
 					}
 				}
 

interpreter/rt_statements.go

@@ -579,6 +579,13 @@ func (rt *tryRuntime) Eval(vs parser.Scope, is map[string]interface{}, tid uint6
 				errObj["data"] = rtError.Data
 			}
 
+			if te, ok := err.(util.TraceableRuntimeError); ok {
+
+				if ts := te.GetTraceString(); ts != nil {
+					errObj["trace"] = ts
+				}
+			}
+
 			res = nil
 
 			for i := 1; i < len(rt.node.Children); i++ {

interpreter/rt_statements_test.go

@@ -955,6 +955,9 @@ error: Something happened: {
   "line": 4,
   "pos": 5,
   "source": "ECALTestRuntime",
+  "trace": [
+    "raise(\"test 12\", null, [1, 2, 3]) (ECALEvalTest:4)"
+  ],
   "type": "test 12"
 }
 Cleanup`[1:] {
@@ -1001,6 +1004,9 @@ error: Something else happened: {
   "line": 4,
   "pos": 5,
   "source": "ECALTestRuntime",
+  "trace": [
+    "raise(\"test 13\", null, [1, 2, 3]) (ECALEvalTest:4)"
+  ],
   "type": "test 13"
 }
 Runtime error: {
@@ -1009,6 +1015,7 @@ Runtime error: {
   "line": 11,
   "pos": 12,
   "source": "ECALTestRuntime",
+  "trace": [],
   "type": "Operand is not a number"
 }
 Cleanup`[1:] {

parser/helper.go

@@ -261,6 +261,7 @@ func (n *ASTNode) ToJSONObject() map[string]interface{} {
 		ret["identifier"] = n.Token.Identifier
 		ret["allowescapes"] = n.Token.AllowEscapes
 		ret["pos"] = n.Token.Pos
+		ret["source"] = n.Token.Lsource
 		ret["line"] = n.Token.Lline
 		ret["linepos"] = n.Token.Lpos
 	}
@@ -334,6 +335,11 @@ func ASTFromJSONObject(jsonAST map[string]interface{}) (*ASTNode, error) {
 		linepos = 0
 	}
 
+	source, ok := jsonAST["source"]
+	if !ok {
+		source = ""
+	}
+
 	// Create meta data
 
 	if meta, ok := jsonAST["meta"]; ok {
@@ -391,6 +397,7 @@ func ASTFromJSONObject(jsonAST map[string]interface{}) (*ASTNode, error) {
 		fmt.Sprint(value),    // Val
 		identifier == true,   // Identifier
 		allowescapes == true, // AllowEscapes
+		fmt.Sprint(source),   // Lsource
 		line,                 // Lline
 		linepos,              // Lpos
 	}

parser/helper_test.go

@@ -16,13 +16,13 @@ import (
 
 func TestASTNode(t *testing.T) {
 
-	n, err := ParseWithRuntime("", "- 1", &DummyRuntimeProvider{})
+	n, err := ParseWithRuntime("test1", "- 1", &DummyRuntimeProvider{})
 	if err != nil {
 		t.Error("Cannot parse test AST:", err)
 		return
 	}
 
-	n2, err := ParseWithRuntime("", "-2", &DummyRuntimeProvider{})
+	n2, err := ParseWithRuntime("test2", "-2", &DummyRuntimeProvider{})
 	if err != nil {
 		t.Error("Cannot parse test AST:", err)
 		return
@@ -40,6 +40,7 @@ Lpos is different 3 vs 2
   "Val": "1",
   "Identifier": false,
   "AllowEscapes": false,
+  "Lsource": "test1",
   "Lline": 1,
   "Lpos": 3
 }
@@ -50,6 +51,7 @@ vs
   "Val": "2",
   "Identifier": false,
   "AllowEscapes": false,
+  "Lsource": "test2",
   "Lline": 1,
   "Lpos": 2
 }
@@ -63,13 +65,13 @@ number: 2
 		return
 	}
 
-	n, err = ParseWithRuntime("", "-1", &DummyRuntimeProvider{})
+	n, err = ParseWithRuntime("test1", "-1", &DummyRuntimeProvider{})
 	if err != nil {
 		t.Error("Cannot parse test AST:", err)
 		return
 	}
 
-	n2, err = ParseWithRuntime("", "-a", &DummyRuntimeProvider{})
+	n2, err = ParseWithRuntime("test2", "-a", &DummyRuntimeProvider{})
 	if err != nil {
 		t.Error("Cannot parse test AST:", err)
 		return
@@ -88,6 +90,7 @@ Identifier is different false vs true
   "Val": "1",
   "Identifier": false,
   "AllowEscapes": false,
+  "Lsource": "test1",
   "Lline": 1,
   "Lpos": 2
 }
@@ -98,6 +101,7 @@ vs
   "Val": "a",
   "Identifier": true,
   "AllowEscapes": false,
+  "Lsource": "test2",
   "Lline": 1,
   "Lpos": 2
 }
@@ -212,6 +216,7 @@ Lpos is different 1 vs 10
   "Val": "1",
   "Identifier": false,
   "AllowEscapes": false,
+  "Lsource": "",
   "Lline": 1,
   "Lpos": 1
 }
@@ -222,6 +227,7 @@ vs
   "Val": "1",
   "Identifier": false,
   "AllowEscapes": false,
+  "Lsource": "",
   "Lline": 1,
   "Lpos": 10
 }

parser/lexer.go

@@ -21,8 +21,8 @@ import (
 	"unicode/utf8"
 )
 
-var namePattern = regexp.MustCompile("^[A-Za-z][A-Za-z0-9]*$")
-var numberPattern = regexp.MustCompile("^[0-9].*$")
+var NamePattern = regexp.MustCompile("^[A-Za-z][A-Za-z0-9]*$")
+var NumberPattern = regexp.MustCompile("^[0-9].*$")
 
 /*
 LexToken represents a token which is returned by the lexer.
@@ -33,6 +33,7 @@ type LexToken struct {
 	Val          string     // Token value
 	Identifier   bool       // Flag if the value is an identifier (not quoted and not a number)
 	AllowEscapes bool       // Flag if the value did interpret escape charaters
+	Lsource      string     // Input source label (e.g. filename)
 	Lline        int        // Line in the input this token appears
 	Lpos         int        // Position in the input line this token appears
 }
@@ -47,6 +48,7 @@ func NewLexTokenInstance(t LexToken) *LexToken {
 		t.Val,
 		t.Identifier,
 		t.AllowEscapes,
+		t.Lsource,
 		t.Lline,
 		t.Lpos,
 	}
@@ -411,7 +413,7 @@ func (l *lexer) emitToken(t LexTokenID) {
 	}
 
 	if l.tokens != nil {
-		l.tokens <- LexToken{t, l.start, l.input[l.start:l.pos], false, false,
+		l.tokens <- LexToken{t, l.start, l.input[l.start:l.pos], false, false, l.name,
 			l.line + 1, l.start - l.lastnl + 1}
 	}
 }
@@ -421,7 +423,7 @@ emitTokenAndValue passes a token with a given value back to the client.
 */
 func (l *lexer) emitTokenAndValue(t LexTokenID, val string, identifier bool, allowEscapes bool) {
 	if l.tokens != nil {
-		l.tokens <- LexToken{t, l.start, val, identifier, allowEscapes, l.line + 1, l.start - l.lastnl + 1}
+		l.tokens <- LexToken{t, l.start, val, identifier, allowEscapes, l.name, l.line + 1, l.start - l.lastnl + 1}
 	}
 }
 
@@ -430,7 +432,7 @@ emitError passes an error token back to the client.
 */
 func (l *lexer) emitError(msg string) {
 	if l.tokens != nil {
-		l.tokens <- LexToken{TokenError, l.start, msg, false, false, l.line + 1, l.start - l.lastnl + 1}
+		l.tokens <- LexToken{TokenError, l.start, msg, false, false, l.name, l.line + 1, l.start - l.lastnl + 1}
 	}
 }
 
@@ -578,7 +580,7 @@ func lexToken(l *lexer) lexFunc {
 
 	// Check for number
 
-	if numberPattern.MatchString(keywordCandidate) {
+	if NumberPattern.MatchString(keywordCandidate) {
 		_, err := strconv.ParseFloat(keywordCandidate, 64)
 
 		if err == nil {
@@ -613,7 +615,7 @@ func lexToken(l *lexer) lexFunc {
 
 	} else {
 
-		if !namePattern.MatchString(keywordCandidate) {
+		if !NamePattern.MatchString(keywordCandidate) {
 			l.emitError(fmt.Sprintf("Cannot parse identifier '%v'. Identifies may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character", keywordCandidate))
 			return nil
 		}

parser/lexer_test.go

@@ -82,6 +82,7 @@ Lpos is different 1 vs 2
   "Val": "not",
   "Identifier": false,
   "AllowEscapes": false,
+  "Lsource": "mytest",
   "Lline": 1,
   "Lpos": 1
 }
@@ -92,6 +93,7 @@ vs
   "Val": "test",
   "Identifier": true,
   "AllowEscapes": false,
+  "Lsource": "mytest",
   "Lline": 2,
   "Lpos": 2
 }` {

parser/runtime.go

@@ -73,4 +73,9 @@ type Scope interface {
 	   String returns a string representation of this scope.
 	*/
 	String() string
+
+	/*
+		ToJSONObject returns this ASTNode and all its children as a JSON object.
+	*/
+	ToJSONObject() map[string]interface{}
 }

scope/varsscope.go

@@ -12,6 +12,7 @@ package scope
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"sort"
 	"strconv"
@@ -29,7 +30,7 @@ type varsScope struct {
 	parent   parser.Scope           // Parent scope
 	children []*varsScope           // Children of this scope (only if tracking is enabled)
 	storage  map[string]interface{} // Storage for variables
-	lock     sync.RWMutex           // Lock for this scope
+	lock     *sync.RWMutex          // Lock for this scope
 }
 
 /*
@@ -44,7 +45,7 @@ NewScopeWithParent creates a new variable scope with a parent. This can be
 used to create scope structures without children links.
 */
 func NewScopeWithParent(name string, parent parser.Scope) parser.Scope {
-	res := &varsScope{name, nil, nil, make(map[string]interface{}), sync.RWMutex{}}
+	res := &varsScope{name, nil, nil, make(map[string]interface{}), &sync.RWMutex{}}
 	SetParentOfScope(res, parent)
 	return res
 }
@@ -54,8 +55,17 @@ SetParentOfScope sets the parent of a given scope. This assumes that the given s
 is a varsScope.
 */
 func SetParentOfScope(scope parser.Scope, parent parser.Scope) {
-	if vs, ok := scope.(*varsScope); ok {
-		vs.parent = parent
+	if pvs, ok := parent.(*varsScope); ok {
+		if vs, ok := scope.(*varsScope); ok {
+
+			vs.lock.Lock()
+			defer vs.lock.Unlock()
+			pvs.lock.Lock()
+			defer pvs.lock.Unlock()
+
+			vs.parent = parent
+			vs.lock = pvs.lock
+		}
 	}
 }
 
@@ -65,6 +75,9 @@ by the parent scope. This means it should not be used for global scopes with
 many children.
 */
 func (s *varsScope) NewChild(name string) parser.Scope {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
 	for _, c := range s.children {
 		if c.name == name {
 			return c
@@ -73,6 +86,7 @@ func (s *varsScope) NewChild(name string) parser.Scope {
 
 	child := NewScope(name).(*varsScope)
 	child.parent = s
+	child.lock = s.lock
 	s.children = append(s.children, child)
 
 	return child
@@ -344,6 +358,21 @@ func (s *varsScope) String() string {
 	return s.scopeStringParents(s.scopeStringChildren())
 }
 
+/*
+ToJSONObject returns this ASTNode and all its children as a JSON object.
+*/
+func (s *varsScope) ToJSONObject() map[string]interface{} {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	var ret map[string]interface{}
+
+	bytes, _ := json.Marshal(s.storage)
+	json.Unmarshal(bytes, &ret)
+
+	return ret
+}
+
 /*
 scopeStringChildren returns a string representation of all children scopes.
 */

scope/varsscope_test.go

@@ -11,6 +11,7 @@
 package scope
 
 import (
+	"encoding/json"
 	"fmt"
 	"testing"
 )
@@ -319,6 +320,12 @@ func TestVarScopeDump(t *testing.T) {
 		return
 	}
 
+	bytes, _ := json.Marshal(sinkVs1.ToJSONObject())
+	if res := string(bytes); res != `{"b":2}` {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
 	if res := sinkVs2.String(); res != `global {
     0 (int) : 0
     sink: 2 {

util/error.go

@@ -20,16 +20,39 @@ import (
 	"devt.de/krotik/ecal/parser"
 )
 
+/*
+TraceableRuntimeError can record and show a stack trace.
+*/
+type TraceableRuntimeError interface {
+	error
+
+	/*
+		AddTrace adds a trace step.
+	*/
+	AddTrace(*parser.ASTNode)
+
+	/*
+		GetTrace returns the current stacktrace.
+	*/
+	GetTrace() []*parser.ASTNode
+
+	/*
+		GetTrace returns the current stacktrace as a string.
+	*/
+	GetTraceString() []string
+}
+
 /*
 RuntimeError is a runtime related error.
 */
 type RuntimeError struct {
-	Source string          // Name of the source which was given to the parser
-	Type   error           // Error type (to be used for equal checks)
-	Detail string          // Details of this error
-	Node   *parser.ASTNode // AST Node where the error occurred
-	Line   int             // Line of the error
-	Pos    int             // Position of the error
+	Source string            // Name of the source which was given to the parser
+	Type   error             // Error type (to be used for equal checks)
+	Detail string            // Details of this error
+	Node   *parser.ASTNode   // AST Node where the error occurred
+	Line   int               // Line of the error
+	Pos    int               // Position of the error
+	Trace  []*parser.ASTNode // Stacktrace
 }
 
 /*
@@ -62,9 +85,9 @@ NewRuntimeError creates a new RuntimeError object.
 */
 func NewRuntimeError(source string, t error, d string, node *parser.ASTNode) error {
 	if node.Token != nil {
-		return &RuntimeError{source, t, d, node, node.Token.Lline, node.Token.Lpos}
+		return &RuntimeError{source, t, d, node, node.Token.Lline, node.Token.Lpos, nil}
 	}
-	return &RuntimeError{source, t, d, node, 0, 0}
+	return &RuntimeError{source, t, d, node, 0, 0, nil}
 }
 
 /*
@@ -83,6 +106,32 @@ func (re *RuntimeError) Error() string {
 	return ret
 }
 
+/*
+AddTrace adds a trace step.
+*/
+func (re *RuntimeError) AddTrace(n *parser.ASTNode) {
+	re.Trace = append(re.Trace, n)
+}
+
+/*
+GetTrace returns the current stacktrace.
+*/
+func (re *RuntimeError) GetTrace() []*parser.ASTNode {
+	return re.Trace
+}
+
+/*
+GetTrace returns the current stacktrace as a string.
+*/
+func (re *RuntimeError) GetTraceString() []string {
+	res := []string{}
+	for _, t := range re.GetTrace() {
+		pp, _ := parser.PrettyPrint(t)
+		res = append(res, fmt.Sprintf("%v (%v:%v)", pp, t.Token.Lsource, t.Token.Lline))
+	}
+	return res
+}
+
 /*
 RuntimeErrorWithDetail is a runtime error with additional environment information.
 */

util/error_test.go

@@ -12,6 +12,7 @@ package util
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 
 	"devt.de/krotik/ecal/parser"
@@ -36,4 +37,23 @@ func TestRuntimeError(t *testing.T) {
 		t.Error("Unexpected result:", err2)
 		return
 	}
+
+	ast, _ = parser.Parse("foo", "a:=1")
+	err3 := NewRuntimeError("foo", fmt.Errorf("foo"), "bar", ast)
+
+	ast, _ = parser.Parse("bar1", "print(b)")
+	err3.(TraceableRuntimeError).AddTrace(ast)
+	ast, _ = parser.Parse("bar2", "raise(c)")
+	err3.(TraceableRuntimeError).AddTrace(ast)
+	ast, _ = parser.Parse("bar3", "1 + d")
+	err3.(TraceableRuntimeError).AddTrace(ast)
+
+	trace := strings.Join(err3.(TraceableRuntimeError).GetTraceString(), "\n")
+
+	if trace != `print(b) (bar1:1)
+raise(c) (bar2:1)
+1 + d (bar3:1)` {
+		t.Error("Unexpected result:", trace)
+		return
+	}
 }

util/types.go

@@ -10,7 +10,9 @@
 
 package util
 
-import "devt.de/krotik/ecal/parser"
+import (
+	"devt.de/krotik/ecal/parser"
+)
 
 /*
 Processor models a top level execution instance for ECAL.
@@ -69,3 +71,110 @@ type Logger interface {
 	*/
 	LogDebug(v ...interface{})
 }
+
+/*
+ContType represents a way how to resume code execution of a suspended thread.
+*/
+type ContType int
+
+/*
+Available lexer token types
+*/
+const (
+	Resume   ContType = iota // Resume code execution until the next breakpoint or the end
+	StepIn                   // Step into a function call or over the next non-function call
+	StepOver                 // Step over the current statement onto the next line
+	StepOut                  // Step out of the current function call
+)
+
+/*
+ECALDebugger is a debugging object which can be used to inspect and modify a running
+ECAL environment.
+*/
+type ECALDebugger interface {
+
+	/*
+		HandleInput handles a given debug instruction. It must be possible to
+		convert the output data into a JSON string.
+	*/
+	HandleInput(input string) (interface{}, error)
+
+	/*
+	   Break on the start of the next execution.
+	*/
+	BreakOnStart(flag bool)
+
+	/*
+	   VisitState is called for every state during the execution of a program.
+	*/
+	VisitState(node *parser.ASTNode, vs parser.Scope, tid uint64) TraceableRuntimeError
+
+	/*
+	   VisitStepInState is called before entering a function call.
+	*/
+	VisitStepInState(node *parser.ASTNode, vs parser.Scope, tid uint64) TraceableRuntimeError
+
+	/*
+	   VisitStepOutState is called after returning from a function call.
+	*/
+	VisitStepOutState(node *parser.ASTNode, vs parser.Scope, tid uint64) TraceableRuntimeError
+
+	/*
+	   SetBreakPoint sets a break point.
+	*/
+	SetBreakPoint(source string, line int)
+
+	/*
+	   DisableBreakPoint disables a break point but keeps the code reference.
+	*/
+	DisableBreakPoint(source string, line int)
+
+	/*
+	   RemoveBreakPoint removes a break point.
+	*/
+	RemoveBreakPoint(source string, line int)
+
+	/*
+		ExtractValue copies a value from a suspended thread into the
+		global variable scope.
+	*/
+	ExtractValue(threadId uint64, varName string, destVarName string) error
+
+	/*
+		InjectValue copies a value from an expression (using the global
+		variable scope) into a suspended thread.
+	*/
+	InjectValue(threadId uint64, varName string, expression string) error
+
+	/*
+	   Continue will continue a suspended thread.
+	*/
+	Continue(threadId uint64, contType ContType)
+
+	/*
+		Status returns the current status of the debugger.
+	*/
+	Status() interface{}
+
+	/*
+	   Describe decribes a thread currently observed by the debugger.
+	*/
+	Describe(threadId uint64) interface{}
+}
+
+/*
+DebugCommand is command which can modify and interrogate the debugger.
+*/
+type DebugCommand interface {
+
+	/*
+		Execute the debug command and return its result. It must be possible to
+		convert the output data into a JSON string.
+	*/
+	Run(debugger ECALDebugger, args []string) (interface{}, error)
+
+	/*
+	   DocString returns a descriptive text about this command.
+	*/
+	DocString() string
+}