feat: Adding plugin support to ECAL

.gitignore

@@ -9,6 +9,7 @@
 /dist
 /build
 /examples/embedding/embedding
+/examples/plugin/*.so
 /ecal-support/node_modules
 /ecal-support/out
 /ecal-support/*.vsix

Makefile

@@ -1,6 +1,9 @@
 export NAME=ecal
 export TAG=`git describe --abbrev=0 --tags`
-export CGO_ENABLED=0
+
+# CGO_ENABLED is enabled here to support Go plugins
+# if Go plugins are not used this can be disabled.
+export CGO_ENABLED=1
 export GOOS=linux
 
 all: build
@@ -29,16 +32,16 @@ build: clean mod generate fmt vet
 	go build -ldflags "-s -w" -o $(NAME) cli/*.go
 
 build-mac: clean mod generate fmt vet
-	GOOS=darwin GOARCH=amd64 go build -o $(NAME).mac cli/*.go
+	CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o $(NAME).mac cli/*.go
 
 build-win: clean mod generate fmt vet
-	GOOS=windows GOARCH=amd64 go build -o $(NAME).exe cli/*.go
+	CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o $(NAME).exe cli/*.go
 
 build-arm7: clean mod generate fmt vet
-	GOOS=linux GOARCH=arm GOARM=7 go build -o $(NAME).arm7 cli/*.go
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o $(NAME).arm7 cli/*.go
 
 build-arm8: clean mod generate fmt vet
-	GOOS=linux GOARCH=arm64 go build -o $(NAME).arm8 cli/*.go
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o $(NAME).arm8 cli/*.go
 
 dist: build build-win build-mac build-arm7 build-arm8
 	rm -fR dist

cli/tool/interpret.go

@@ -11,11 +11,13 @@
 package tool
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"devt.de/krotik/common/fileutil"
@@ -54,7 +56,8 @@ type CLIInterpreter struct {
 	CustomWelcomeMessage string
 	CustomHelpString     string
 
-	EntryFile string // Entry file for the program
+	EntryFile   string // Entry file for the program
+	LoadPlugins bool   // Flag if stdlib plugins should be loaded
 
 	// Parameter these can either be set programmatically or via CLI args
 
@@ -75,7 +78,8 @@ type CLIInterpreter struct {
 NewCLIInterpreter creates a new commandline interpreter for ECAL.
 */
 func NewCLIInterpreter() *CLIInterpreter {
-	return &CLIInterpreter{scope.NewScope(scope.GlobalScope), nil, nil, "", "", "", nil, nil, nil, nil, os.Stdout}
+	return &CLIInterpreter{scope.NewScope(scope.GlobalScope), nil, nil, "", "", "",
+		true, nil, nil, nil, nil, os.Stdout}
 }
 
 /*
@@ -225,63 +229,105 @@ func (i *CLIInterpreter) Interpret(interactive bool) error {
 		return nil
 	}
 
-	err := i.CreateTerm()
+	err := i.LoadStdlibPlugins(interactive)
 
-	if interactive {
-		fmt.Fprintln(i.LogOut, fmt.Sprintf("ECAL %v", config.ProductVersion))
-	}
+	if err == nil {
+		err = i.CreateTerm()
 
-	// Create Runtime Provider
+		if interactive {
+			fmt.Fprintln(i.LogOut, fmt.Sprintf("ECAL %v", config.ProductVersion))
+		}
 
-	if err == nil {
+		// Create Runtime Provider
 
-		if err = i.CreateRuntimeProvider("console"); err == nil {
+		if err == nil {
 
-			tid := i.RuntimeProvider.NewThreadID()
+			if err = i.CreateRuntimeProvider("console"); err == nil {
 
-			if interactive {
-				if lll, ok := i.RuntimeProvider.Logger.(*util.LogLevelLogger); ok {
-					fmt.Fprint(i.LogOut, fmt.Sprintf("Log level: %v - ", lll.Level()))
-				}
+				tid := i.RuntimeProvider.NewThreadID()
 
-				fmt.Fprintln(i.LogOut, fmt.Sprintf("Root directory: %v", *i.Dir))
+				if interactive {
+					if lll, ok := i.RuntimeProvider.Logger.(*util.LogLevelLogger); ok {
+						fmt.Fprint(i.LogOut, fmt.Sprintf("Log level: %v - ", lll.Level()))
+					}
 
-				if i.CustomWelcomeMessage != "" {
-					fmt.Fprintln(i.LogOut, fmt.Sprintf(i.CustomWelcomeMessage))
+					fmt.Fprintln(i.LogOut, fmt.Sprintf("Root directory: %v", *i.Dir))
+
+					if i.CustomWelcomeMessage != "" {
+						fmt.Fprintln(i.LogOut, fmt.Sprintf(i.CustomWelcomeMessage))
+					}
 				}
-			}
 
-			// Execute file if given
+				// Execute file if given
 
-			if err = i.LoadInitialFile(tid); err == nil {
+				if err = i.LoadInitialFile(tid); err == nil {
 
-				// Drop into interactive shell
+					// Drop into interactive shell
 
-				if interactive {
+					if interactive {
 
-					// Add history functionality without file persistence
+						// Add history functionality without file persistence
 
-					i.Term, err = termutil.AddHistoryMixin(i.Term, "",
-						func(s string) bool {
-							return i.isExitLine(s)
-						})
+						i.Term, err = termutil.AddHistoryMixin(i.Term, "",
+							func(s string) bool {
+								return i.isExitLine(s)
+							})
 
-					if err == nil {
+						if err == nil {
 
-						if err = i.Term.StartTerm(); err == nil {
-							var line string
+							if err = i.Term.StartTerm(); err == nil {
+								var line string
 
-							defer i.Term.StopTerm()
+								defer i.Term.StopTerm()
 
-							fmt.Fprintln(i.LogOut, "Type 'q' or 'quit' to exit the shell and '?' to get help")
+								fmt.Fprintln(i.LogOut, "Type 'q' or 'quit' to exit the shell and '?' to get help")
 
-							line, err = i.Term.NextLine()
-							for err == nil && !i.isExitLine(line) {
-								trimmedLine := strings.TrimSpace(line)
+								line, err = i.Term.NextLine()
+								for err == nil && !i.isExitLine(line) {
+									trimmedLine := strings.TrimSpace(line)
 
-								i.HandleInput(i.Term, trimmedLine, tid)
+									i.HandleInput(i.Term, trimmedLine, tid)
 
-								line, err = i.Term.NextLine()
+									line, err = i.Term.NextLine()
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return err
+}
+
+/*
+LoadStdlibPlugins load plugins from .ecal.json.
+*/
+func (i *CLIInterpreter) LoadStdlibPlugins(interactive bool) error {
+	var err error
+
+	if i.LoadPlugins {
+		confFile := filepath.Join(*i.Dir, ".ecal.json")
+		if ok, _ := fileutil.PathExists(confFile); ok {
+
+			if interactive {
+				fmt.Fprintln(i.LogOut, fmt.Sprintf("Loading stdlib plugins from %v", confFile))
+			}
+
+			var content []byte
+			if content, err = ioutil.ReadFile(confFile); err == nil {
+				var conf map[string]interface{}
+				if err = json.Unmarshal(content, &conf); err == nil {
+					if stdlibPlugins, ok := conf["stdlibPlugins"]; ok {
+						err = fmt.Errorf("Config stdlibPlugins should be a list")
+						if plugins, ok := stdlibPlugins.([]interface{}); ok {
+							err = nil
+							if errs := stdlib.LoadStdlibPlugins(plugins); len(errs) > 0 {
+								for _, e := range errs {
+									fmt.Fprintln(i.LogOut, fmt.Sprintf("Error loading plugins: %v", e))
+								}
+								err = fmt.Errorf("Could not load plugins defined in .ecal.json")
 							}
 						}
 					}

cli/tool/interpret_test.go

@@ -52,7 +52,6 @@ func newTestInterpreter() *CLIInterpreter {
 func newTestInterpreterWithConfig() *CLIInterpreter {
 	tin := newTestInterpreter()
 
-	// Setup
 	if res, _ := fileutil.PathExists(testDir); res {
 		os.RemoveAll(testDir)
 	}
@@ -68,8 +67,6 @@ func newTestInterpreterWithConfig() *CLIInterpreter {
 
 	tin.CustomWelcomeMessage = "123"
 
-	// Teardown
-
 	return tin
 }
 
@@ -144,6 +141,41 @@ func TestInterpretBasicFunctions(t *testing.T) {
 		t.Error("Unexpected entryfile:", tin.EntryFile)
 		return
 	}
+
+	flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) // Reset CLI parsing
+
+	osArgs = []string{"foo", "bar"}
+
+	// Try to load non-existing plugins (success case is tested in stdlib)
+
+	tin = newTestInterpreterWithConfig()
+	defer tearDown()
+
+	l1 := ""
+	tin.LogFile = &l1
+	l2 := ""
+	tin.LogLevel = &l2
+
+	ioutil.WriteFile(filepath.Join(testDir, ".ecal.json"), []byte(`{
+  "stdlibPlugins" : [{
+    "package" : "mypkg",
+    "name" : "myfunc",
+    "path" : "./myfunc.so",
+    "symbol" : "ECALmyfunc"
+  }]
+}`), 0666)
+
+	err := tin.Interpret(true)
+
+	if err == nil || err.Error() != "Could not load plugins defined in .ecal.json" {
+		t.Error("Unexpected result:", err.Error())
+		return
+	}
+
+	if !strings.Contains(testLogOut.String(), "Error loading plugins") {
+		t.Error("Unexpected result:", testLogOut.String())
+		return
+	}
 }
 
 func TestCreateRuntimeProvider(t *testing.T) {

examples/plugin/.ecal.json

@@ -0,0 +1,8 @@
+{
+  "stdlibPlugins" : [{
+    "package" : "mypkg",
+    "name" : "myfunc",
+    "path" : "./myfunc.so",
+    "symbol" : "ECALmyfunc"
+  }]
+}

examples/plugin/buildplugin.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+go build -ldflags "-s -w" -buildmode=plugin -o myfunc.so myfunc.go

examples/plugin/greeting.ecal

@@ -0,0 +1,9 @@
+a := "Hans"
+res := mypkg.myfunc(a)
+log(res)
+
+try {
+  mypkg.myfunc()
+} except e {
+  error("Got an error: ", e.detail)
+}

examples/plugin/myfunc.go

@@ -0,0 +1,61 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+/*
+Example ECAL stdlib function plugin.
+
+The plugins must be valid Go plugins: https://golang.org/pkg/plugin
+
+ECAL usable functions imported via a plugin must conform to the following interface:
+
+type ECALPluginFunction interface {
+	Run(args []interface{}) (interface{}, error) // Function execution with given arguments
+	DocString() string // Returns some function description
+}
+*/
+
+package main
+
+import "fmt"
+
+func init() {
+
+	// Here goes some initialisation code
+
+	Greeting = "Hello"
+	ECALmyfunc = myfunc{"World"}
+}
+
+/*
+Greeting is first word in the output
+*/
+var Greeting string
+
+type myfunc struct {
+	place string
+}
+
+func (f *myfunc) Run(args []interface{}) (interface{}, error) {
+	if len(args) == 0 {
+		return "", fmt.Errorf("Need a name to greet as argument")
+	}
+
+	return fmt.Sprintf("%v %v for %v", Greeting, f.place, args[0]), nil
+}
+
+func (f *myfunc) DocString() string {
+	return "Myfunc is an example function"
+}
+
+// Exported bits
+
+/*
+ECALmyfunc is the exported function which can be used by ECAL
+*/
+var ECALmyfunc myfunc

examples/plugin/run.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+../../ecal run greeting.ecal

go.mod

@@ -2,4 +2,4 @@ module devt.de/krotik/ecal
 
 go 1.12
 
-require devt.de/krotik/common v1.3.9
+require devt.de/krotik/common v1.4.0

go.sum

@@ -1,2 +1,2 @@
-devt.de/krotik/common v1.3.9 h1:/DdNkkaaplUXHeA+6juY3f+WMUOkc5zycar2TdPXHB0=
-devt.de/krotik/common v1.3.9/go.mod h1:X4nsS85DAxyHkwSg/Tc6+XC2zfmGeaVz+37F61+eSaI=
+devt.de/krotik/common v1.4.0 h1:chZihshmuv1yehyujrYyW7Yg4cgRqqIWEG2IAzhfFkA=
+devt.de/krotik/common v1.4.0/go.mod h1:X4nsS85DAxyHkwSg/Tc6+XC2zfmGeaVz+37F61+eSaI=

stdlib/stdlib.go

@@ -12,14 +12,35 @@ package stdlib
 
 import (
 	"fmt"
+	"plugin"
+	"reflect"
 	"strings"
 
 	"devt.de/krotik/ecal/util"
 )
 
+/*
+internalStdlibFuncMap holds all registered functions
+*/
 var internalStdlibFuncMap = make(map[string]util.ECALFunction)
+
+/*
+internalStdlibDocMap holds the docstrings for all registered functions
+*/
 var internalStdlibDocMap = make(map[string]string)
 
+/*
+pluginLookup is an interface for required function of the plugin object - only used for unit testing.
+*/
+type pluginLookup interface {
+	Lookup(symName string) (plugin.Symbol, error)
+}
+
+/*
+pluginTestLookup override plugin object - only used for unit testing.
+*/
+var pluginTestLookup pluginLookup
+
 /*
 AddStdlibPkg adds a package to stdlib. A package needs to be added before functions
 can be added.
@@ -53,6 +74,76 @@ func AddStdlibFunc(pkg string, name string, funcObj util.ECALFunction) error {
 	return nil
 }
 
+/*
+LoadStdlibPlugins attempts to load stdlib functions from a given list of definitions.
+*/
+func LoadStdlibPlugins(jsonObj []interface{}) []error {
+	var errs []error
+
+	for _, i := range jsonObj {
+		if err := LoadStdlibPlugin(i.(map[string]interface{})); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	return errs
+}
+
+/*
+LoadStdlibPlugin attempts to load a stdlib function from a given definition.
+*/
+func LoadStdlibPlugin(jsonObj map[string]interface{}) error {
+	pkg := fmt.Sprint(jsonObj["package"])
+	name := fmt.Sprint(jsonObj["name"])
+	path := fmt.Sprint(jsonObj["path"])
+	symName := fmt.Sprint(jsonObj["symbol"])
+
+	return AddStdlibPluginFunc(pkg, name, path, symName)
+}
+
+/*
+AddStdlibPluginFunc adds a function to stdlib via a loaded plugin.
+The plugin needs to be build as a Go plugin (https://golang.org/pkg/plugin):
+
+go build -buildmode=plugin -o myfunc.so myfunc.go
+
+And have an exported variable (passed here as symName) which conforms
+to util.ECALPluginFunction.
+*/
+func AddStdlibPluginFunc(pkg string, name string, path string, symName string) error {
+	var err error
+	var plug pluginLookup
+
+	AddStdlibPkg(pkg, "Functions provided by plugins")
+
+	if plug, err = plugin.Open(path); err == nil || pluginTestLookup != nil {
+		var sym plugin.Symbol
+
+		if pluginTestLookup != nil {
+			plug = pluginTestLookup
+		}
+
+		if sym, err = plug.Lookup(symName); err == nil {
+
+			if stdlibPluginFunc, ok := sym.(util.ECALPluginFunction); ok {
+
+				adapterFunc := func(a ...interface{}) (interface{}, error) {
+					return stdlibPluginFunc.Run(a)
+				}
+
+				err = AddStdlibFunc(pkg, name, &ECALFunctionAdapter{
+					reflect.ValueOf(adapterFunc), stdlibPluginFunc.DocString()})
+
+			} else {
+
+				err = fmt.Errorf("Symbol %v is not a stdlib function", symName)
+			}
+		}
+	}
+
+	return err
+}
+
 /*
 GetStdlibSymbols returns all available packages of stdlib and their constant
 and function symbols.

stdlib/stdlib_test.go

@@ -13,7 +13,9 @@ package stdlib
 import (
 	"fmt"
 	"math"
+	"plugin"
 	"reflect"
+	"strings"
 	"testing"
 )
 
@@ -129,3 +131,129 @@ func TestAddStdLibFunc(t *testing.T) {
 		return
 	}
 }
+
+func TestAddPluginStdLibFunc(t *testing.T) {
+	var err error
+
+	// Uncomment the commented parts in this test to run against an
+	// actual compiled plugin in the examples/plugin directory
+
+	/*
+		err = AddStdlibPluginFunc("foo", "bar", filepath.Join("..", "examples", "plugin", "myfunc.so"), "ECALmyfunc")
+
+		if err != nil {
+			t.Error("Unexpected result:", err)
+			return
+		}
+	*/
+
+	pluginTestLookup = &testLookup{&testECALPluginFunction{}, nil}
+
+	err = AddStdlibPluginFunc("foo", "bar", "", "ECALmyfunc")
+
+	pluginTestLookup = nil
+
+	if err != nil {
+		t.Error("Unexpected result:", err)
+		return
+	}
+
+	pluginTestLookup = &testLookup{&testECALPluginFunction{}, nil}
+
+	errs := LoadStdlibPlugins([]interface{}{
+		map[string]interface{}{
+			"package": "foo",
+			"name":    "bar",
+			"path":    "",
+			"symbol":  "ECALmyfunc",
+		},
+		map[string]interface{}{
+			"package": "foo",
+			"name":    "bar",
+			"path":    "",
+			"symbol":  "showerror",
+		},
+	})
+
+	pluginTestLookup = nil
+
+	if fmt.Sprint(errs) != "[Test lookup error]" {
+		t.Error("Unexpected result:", errs)
+		return
+	}
+
+	pfunc, ok := GetStdlibFunc("foo.bar")
+
+	if !ok {
+		t.Error("Unexpected result:", pfunc, ok)
+		return
+	}
+
+	res, err := pfunc.Run("", nil, nil, 0, []interface{}{"John"})
+
+	if err != nil || res != "Hello World for John" {
+		t.Error("Unexpected result:", res, err)
+		return
+	}
+
+	// Test errors
+
+	/*
+		err = AddStdlibPluginFunc("foo", "bar", filepath.Join("..", "examples", "plugin", "myfunc.so"), "Greeting")
+
+		if err == nil || err.Error() != "Symbol Greeting is not a stdlib function" {
+			t.Error("Unexpected result:", err)
+			return
+		}
+
+		err = AddStdlibPluginFunc("foo", "bar", filepath.Join("..", "examples", "plugin", "myfunc.so"), "foo")
+
+		if err == nil || !strings.Contains(err.Error(), "symbol foo not found") {
+			t.Error("Unexpected result:", err)
+			return
+		}
+	*/
+
+	pluginTestLookup = &testLookup{"foo", nil}
+	err = AddStdlibPluginFunc("foo", "bar", "", "Greeting")
+
+	if err == nil || err.Error() != "Symbol Greeting is not a stdlib function" {
+		t.Error("Unexpected result:", err)
+		return
+	}
+
+	pluginTestLookup = &testLookup{nil, fmt.Errorf("symbol foo not found")}
+	err = AddStdlibPluginFunc("foo", "bar", "", "foo")
+
+	if err == nil || !strings.Contains(err.Error(), "symbol foo not found") {
+		t.Error("Unexpected result:", err)
+		return
+	}
+}
+
+type testLookup struct {
+	ret interface{}
+	err error
+}
+
+func (tl *testLookup) Lookup(symName string) (plugin.Symbol, error) {
+	if symName == "showerror" {
+		return nil, fmt.Errorf("Test lookup error")
+	}
+	return tl.ret, tl.err
+}
+
+type testECALPluginFunction struct {
+}
+
+func (tf *testECALPluginFunction) Run(args []interface{}) (interface{}, error) {
+	if len(args) == 0 {
+		return "", fmt.Errorf("Need a name to greet as argument")
+	}
+
+	return fmt.Sprintf("Hello World for %v", args[0]), nil
+}
+
+func (tf *testECALPluginFunction) DocString() string {
+	return "Myfunc is an example function"
+}

util/types.go

@@ -53,6 +53,22 @@ type ECALFunction interface {
 	DocString() (string, error)
 }
 
+/*
+ECALPluginFunction models a callable function in ECAL which can be imported via a plugin.
+*/
+type ECALPluginFunction interface {
+
+	/*
+		Run executes this function with a given list of arguments.
+	*/
+	Run(args []interface{}) (interface{}, error)
+
+	/*
+	   DocString returns a descriptive text about this function.
+	*/
+	DocString() string
+}
+
 /*
 Logger is required external object to which the interpreter releases its log messages.
 */