Merge 8cb50d0b85 into 746392cfd2

cmd/abigen/main.go

@@ -17,7 +17,6 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -81,19 +80,21 @@ func main() {
 		for _, kind := range strings.Split(*excFlag, ",") {
 			exclude[strings.ToLower(kind)] = true
 		}
-		contracts, err := compiler.CompileSolidity(*solcFlag, *solFlag)
+		solc, err := compiler.InitSolc(*solcFlag)
-		if err != nil {
+
+		solReturn, err := solc.Compile(compiler.FlagOpts{}, *solFlag)
+		if err != nil || solReturn.Typ != compiler.Solc {
 			fmt.Printf("Failed to build Solidity contract: %v\n", err)
 			os.Exit(-1)
 		}
 		// Gather all non-excluded contract for binding
-		for name, contract := range contracts {
+		for name, contract := range solReturn.Contracts {
 			if exclude[strings.ToLower(name)] {
 				continue
 			}
-			abi, _ := json.Marshal(contract.Info.AbiDefinition) // Flatten the compiler parse
+
-			abis = append(abis, string(abi))
+			abis = append(abis, contract.Abi)
-			bins = append(bins, contract.Code)
+			bins = append(bins, contract.Bin)
 
 			nameParts := strings.Split(name, ":")
 			types = append(types, nameParts[len(nameParts)-1])

common/compiler/solidity.go

@@ -22,174 +22,207 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os/exec"
+	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
 )
 
 var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
 
-type Contract struct {
+// Initialize a versioned Solc compiler.
-	Code string       `json:"code"`
+func InitSolc(command string) (*Solidity, error) {
-	Info ContractInfo `json:"info"`
+	if command == "" {
+		command = "solc"
+	}
+	if _, err := exec.LookPath(command); err != nil {
+		return nil, fmt.Errorf("compiler: could not find %v in PATH", command)
+	}
+	s := &Solidity{NamedCmd: command}
+	if err := s.version(); err != nil {
+		return nil, err
+	}
+	return s, nil
+
 }
 
-type ContractInfo struct {
+//The following represents solidity outputs from the compiler that we're interested in
-	Source          string      `json:"source"`
+type SolcReturn struct {
-	Language        string      `json:"language"`
+	Warning   string
-	LanguageVersion string      `json:"languageVersion"`
+	Version   string               `json:"version"`
-	CompilerVersion string      `json:"compilerVersion"`
+	Contracts map[string]SolcItems `json:"contracts"`
-	CompilerOptions string      `json:"compilerOptions"`
+}
-	AbiDefinition   interface{} `json:"abiDefinition"`
+
-	UserDoc         interface{} `json:"userDoc"`
+//The key return items to enable unmarshalling from the returns from the compiler
-	DeveloperDoc    interface{} `json:"developerDoc"`
+type SolcItems struct {
-	Metadata        string      `json:"metadata"`
+	Bin      string `json:"bin"`
+	Abi      string `json:"abi"`
+	DevDoc   string `json:"devdoc"`
+	UserDoc  string `json:"userdoc"`
+	Metadata string `json:"metadata"`
 }
 
 // Solidity contains information about the solidity compiler.
 type Solidity struct {
-	Path, Version, FullVersion string
+	NamedCmd, Path, Version, FullVersion string
-	Major, Minor, Patch        int
+	Major, Minor, Patch                  int
 }
 
-// --combined-output format
+//This is a template to define our inputs for the compiler flags
-type solcOutput struct {
+type SolcFlagOpts struct {
-	Contracts map[string]struct {
+	// (Required) what to get in the output, can be any combination of [abi, bin, userdoc, devdoc, metadata]
-		Bin, Abi, Devdoc, Userdoc, Metadata string
+	// abi: application binary interface. Necessary for interaction with contracts.
-	}
+	// bin: binary bytecode. Necessary for creating and deploying and interacting with contracts.
-	Version string
+	// userdoc: natspec for users.
-}
+	// devdoc: natspec for devs.
-
+	// metadata: contract metadata.
-func (s *Solidity) makeArgs() []string {
+	CombinedOutput []string
-	p := []string{
+	// (Optional) Direct string of library address mappings.
-		"--combined-json", "bin,abi,userdoc,devdoc",
+	//  Syntax: <libraryName>:<address>,<libraryName>:<address>
-		"--add-std",  // include standard lib contracts
+	//  Address is interpreted as a hex string optionally prefixed by 0x.
-		"--optimize", // code optimizer switched on
+	Libraries map[string]common.Address
-	}
+	// (Optional) Remappings, see https://solidity.readthedocs.io/en/latest/layout-of-source-files.html#use-in-actual-compilers
-	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
+	// Syntax: <remoteName>=<localName>
-		p[1] += ",metadata"
+	Remappings []string
-	}
+	// (Optional) if true, enable standard library contracts
-	return p
+	StdLib bool
+	// (Optional) if true, optimizes solidity code
+	Optimize bool
+	// (Optional) the number of optimization runs to run on solidity
+	OptimizeRuns uint64
+	// (Optional) For anything else we may have missed, if filled will default override other flags.
+	Exec string
 }
 
 // SolidityVersion runs solc and parses its version output.
-func SolidityVersion(solc string) (*Solidity, error) {
+func (s *Solidity) version() error {
-	if solc == "" {
-		solc = "solc"
-	}
 	var out bytes.Buffer
-	cmd := exec.Command(solc, "--version")
+	cmd := exec.Command(s.NamedCmd, "--version")
 	cmd.Stdout = &out
 	err := cmd.Run()
 	if err != nil {
-		return nil, err
+		return err
 	}
 	matches := versionRegexp.FindStringSubmatch(out.String())
 	if len(matches) != 4 {
-		return nil, fmt.Errorf("can't parse solc version %q", out.String())
+		return fmt.Errorf("%v: can't parse version %q", s.NamedCmd, out.String())
 	}
-	s := &Solidity{Path: cmd.Path, FullVersion: out.String(), Version: matches[0]}
+	s = &Solidity{Path: cmd.Path, FullVersion: out.String(), Version: matches[0]}
 	if s.Major, err = strconv.Atoi(matches[1]); err != nil {
-		return nil, err
+		return err
 	}
 	if s.Minor, err = strconv.Atoi(matches[2]); err != nil {
-		return nil, err
+		return err
 	}
 	if s.Patch, err = strconv.Atoi(matches[3]); err != nil {
-		return nil, err
+		return err
 	}
-	return s, nil
+	return nil
 }
 
-// CompileSolidityString builds and returns all the contracts contained within a source string.
+// Compiles a series of files using the solidity compiler
-func CompileSolidityString(solc, source string) (map[string]*Contract, error) {
+func (s *Solidity) Compile(flags SolcFlagOpts, files ...string) (Return, error) {
-	if len(source) == 0 {
+
-		return nil, errors.New("solc: empty source string")
+	if reflect.DeepEqual(flags.SolcFlagOpts, (SolcFlagOpts{})) {
+		flags.defaultSolcFlagOpts(s)
 	}
-	s, err := SolidityVersion(solc)
+
-	if err != nil {
+	return s.execute(flags.assembleSolcCommand(files...)...)
-		return nil, err
-	}
-	args := append(s.makeArgs(), "--")
-	cmd := exec.Command(s.Path, append(args, "-")...)
-	cmd.Stdin = strings.NewReader(source)
-	return s.run(cmd, source)
 }
 
-// CompileSolidity compiles all given Solidity source files.
+func (s *Solidity) Link(libs map[string]common.Address, binary string) (string, error) {
-func CompileSolidity(solc string, sourcefiles ...string) (map[string]*Contract, error) {
+	var refinedBinary bytes.Buffer
-	if len(sourcefiles) == 0 {
+	var stderr bytes.Buffer
-		return nil, errors.New("solc: no source files")
+
+	buf := bytes.NewBufferString(binary)
+	linkCmd := exec.Command("solc", "--link", "--libraries", stringifyLibs(libs))
+
+	linkCmd.Stdin = buf
+	linkCmd.Stderr = &stderr
+	linkCmd.Stdout = &refinedBinary
+
+	linkCmd.Start()
+	linkCmd.Wait()
+
+	if stderr.String() != "" {
+		return "", errors.New(stderr.String())
 	}
-	source, err := slurpFiles(sourcefiles)
+
-	if err != nil {
+	return refinedBinary.String(), nil
-		return nil, err
-	}
-	s, err := SolidityVersion(solc)
-	if err != nil {
-		return nil, err
-	}
-	args := append(s.makeArgs(), "--")
-	cmd := exec.Command(s.Path, append(args, sourcefiles...)...)
-	return s.run(cmd, source)
 }
 
-func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
+func stringifyLibs(libs map[string]common.Address) string {
+	var combinedLibs []string
+	for x, y := range libs {
+		combinedLibs = append(combinedLibs, x+":"+y.String())
+	}
+	return strings.Join(combinedLibs, ",")
+}
+
+func (f SolcFlagOpts) assembleSolcCommand(files ...string) (command []string) {
+
+	switch {
+	case f.Exec != "":
+		command = append(command, f.Exec)
+	default:
+		if len(f.Remappings) > 0 {
+			command = append(command, strings.Join(f.Remappings, " "))
+		}
+		if len(f.CombinedOutput) > 0 {
+			command = append(command, "--combined-json", strings.Join(f.CombinedOutput, ","))
+		}
+		if len(f.Libraries) > 0 {
+			command = append(command, []string{"--libraries", stringifyLibs(f.Libraries)}...)
+		}
+		if f.Optimize {
+			command = append(command, "--optimize")
+		}
+		if f.StdLib {
+			command = append(command, "--std-lib")
+		}
+		if f.OptimizeRuns != 0 {
+			command = append(command, "--optimize-runs", strconv.FormatUint(f.OptimizeRuns, 10))
+		}
+	}
+	return append(command, files...)
+}
+
+func (f *SolcFlagOpts) defaultSolcFlagOpts(s *Solidity) {
+	f = &SolcFlagOpts{
+		CombinedOutput: []string{"bin", "abi", "userdoc", "devdoc"},
+		StdLib:         true,
+		Optimize:       true,
+	}
+
+	if s.Major >= 0 && s.Minor >= 4 && s.Patch > 6 {
+		f.CombinedOutput = append(f.CombinedOutput, "metadata")
+	}
+
+	return
+}
+
+func (s *Solidity) execute(flagsAndFiles ...string) (SolcReturn, error) {
 	var stderr, stdout bytes.Buffer
+	var output SolcReturn
+
+	cmd := exec.Command(s.NamedCmd, flagsAndFiles...)
 	cmd.Stderr = &stderr
 	cmd.Stdout = &stdout
+
 	if err := cmd.Run(); err != nil {
-		return nil, fmt.Errorf("solc: %v\n%s", err, stderr.Bytes())
+		return Return{}, fmt.Errorf("%v: %v\n%s", s.NamedCmd, err, stderr.Bytes())
-	}
-	var output solcOutput
-	if err := json.Unmarshal(stdout.Bytes(), &output); err != nil {
-		return nil, err
 	}
 
-	// Compilation succeeded, assemble and return the contracts.
+	buf := stdout.Bytes()
-	contracts := make(map[string]*Contract)
-	for name, info := range output.Contracts {
-		// Parse the individual compilation results.
-		var abi interface{}
-		if err := json.Unmarshal([]byte(info.Abi), &abi); err != nil {
-			return nil, fmt.Errorf("solc: error reading abi definition (%v)", err)
-		}
-		var userdoc interface{}
-		if err := json.Unmarshal([]byte(info.Userdoc), &userdoc); err != nil {
-			return nil, fmt.Errorf("solc: error reading user doc: %v", err)
-		}
-		var devdoc interface{}
-		if err := json.Unmarshal([]byte(info.Devdoc), &devdoc); err != nil {
-			return nil, fmt.Errorf("solc: error reading dev doc: %v", err)
-		}
-		contracts[name] = &Contract{
-			Code: "0x" + info.Bin,
-			Info: ContractInfo{
-				Source:          source,
-				Language:        "Solidity",
-				LanguageVersion: s.Version,
-				CompilerVersion: s.Version,
-				CompilerOptions: strings.Join(s.makeArgs(), " "),
-				AbiDefinition:   abi,
-				UserDoc:         userdoc,
-				DeveloperDoc:    devdoc,
-				Metadata:        info.Metadata,
-			},
-		}
-	}
-	return contracts, nil
-}
 
-func slurpFiles(files []string) (string, error) {
+	if err := json.Unmarshal(buf, &output); err != nil {
-	var concat bytes.Buffer
+		return Return{}, err
-	for _, file := range files {
-		content, err := ioutil.ReadFile(file)
-		if err != nil {
-			return "", err
-		}
-		concat.Write(content)
 	}
-	return concat.String(), nil
+
+	output.Warning = string(stderr.Bytes())
+
+	return output, nil
 }

common/compiler/solidity_test.go

@@ -17,20 +17,110 @@
 package compiler
 
 import (
+	"io/ioutil"
+	"os"
 	"os/exec"
+	"path/filepath"
+	"strings"
 	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
 )
 
-const (
+const regularSolFile = `pragma solidity >= 0.0.0;
-	testSource = `
+	contract main {
-contract test {
+
-   /// @notice Will multiply ` + "`a`" + ` by 7.
+		}`
-   function multiply(uint a) returns(uint d) {
+
-       return a * 7;
+const faultySol = `pragma solidity >= 0.0.0;
-   }
+	contract main {
+		uint a;
+		function f() {
+			a = 1;
+		}
+	`
+const solNoPragma = `contract main {
+		uint a;
+		function f() {
+			a = 1;
+		}
+	}`
+const simplyLibrarySol = `pragma solidity >=0.0.0;
+
+library Set {
+  struct Data { mapping(uint => bool) flags; }
+  function insert(Data storage self, uint value)
+      returns (bool)
+  {
+      if (self.flags[value])
+          return false; // already there
+      self.flags[value] = true;
+      return true;
+  }
+
+  function remove(Data storage self, uint value)
+      returns (bool)
+  {
+      if (!self.flags[value])
+          return false; // not there
+      self.flags[value] = false;
+      return true;
+  }
+
+  function contains(Data storage self, uint value)
+      returns (bool)
+  {
+      return self.flags[value];
+  }
 }
-`
+
-)
+contract C {
+    Set.Data knownValues;
+    function register(uint value) {
+        if (!Set.insert(knownValues, value))
+            throw;
+    }
+}`
+
+const solFile1 = `pragma solidity >=0.0.0;
+import "/somedir/set.sol";
+
+contract C {
+    Set.Data knownValues;
+    function register(uint value) {
+        if (!Set.insert(knownValues, value))
+            throw;
+    }
+}`
+
+const solFile2 = `pragma solidity >=0.0.0;
+
+library Set {
+  struct Data { mapping(uint => bool) flags; }
+  function insert(Data storage self, uint value)
+      returns (bool)
+  {
+      if (self.flags[value])
+          return false; // already there
+      self.flags[value] = true;
+      return true;
+  }
+
+  function remove(Data storage self, uint value)
+      returns (bool)
+  {
+      if (!self.flags[value])
+          return false; // not there
+      self.flags[value] = false;
+      return true;
+  }
+
+  function contains(Data storage self, uint value)
+      returns (bool)
+  {
+      return self.flags[value];
+  }
+}`
 
 func skipWithoutSolc(t *testing.T) {
 	if _, err := exec.LookPath("solc"); err != nil {
@@ -38,40 +128,213 @@ func skipWithoutSolc(t *testing.T) {
 	}
 }
 
-func TestCompiler(t *testing.T) {
+func writeToTempFile(tmpfile *os.File, content []byte) error {
+
+	if _, err := tmpfile.Write(content); err != nil {
+		return err
+	}
+	if err := tmpfile.Close(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func TestSolcCompilerNormal(t *testing.T) {
+
 	skipWithoutSolc(t)
 
-	contracts, err := CompileSolidityString("", testSource)
+	solc, err := InitSolc("solc")
 	if err != nil {
-		t.Fatalf("error compiling source. result %v: %v", contracts, err)
+		t.Fatalf("Could not initialize solc: %v", err)
 	}
-	if len(contracts) != 1 {
+
-		t.Errorf("one contract expected, got %d", len(contracts))
+	content := []byte(regularSolFile)
+	tmpfile, err := ioutil.TempFile("", "simpleContract.sol")
+	if err != nil {
+		t.Fatal(err)
 	}
-	c, ok := contracts["test"]
+	defer os.Remove(tmpfile.Name()) // clean up
-	if !ok {
+
-		c, ok = contracts["<stdin>:test"]
+	err = writeToTempFile(tmpfile, content)
-		if !ok {
+	if err != nil {
-			t.Fatal("info for contract 'test' not present in result")
+		t.Fatal(err)
-		}
 	}
-	if c.Code == "" {
+
-		t.Error("empty code")
+	flags := SolcFlagOpts{
+		CombinedOutput: []string{"bin", "abi"},
 	}
-	if c.Info.Source != testSource {
+
-		t.Error("wrong source")
+	solReturn, err := solc.Compile(FlagOpts{flags}, tmpfile.Name())
+	if err != nil {
+		t.Errorf("Expected no errors: %v", err)
 	}
-	if c.Info.CompilerVersion == "" {
+
-		t.Error("empty version")
+	if solReturn.Warning != "" || len(solReturn.Contracts) != 1 {
+		t.Fatalf("Expected no warnings and expected 1 contract item. Got %v for warnings, and %v for contract items", solReturn.Warning, len(solReturn.Contracts))
 	}
 }
 
-func TestCompileError(t *testing.T) {
+func TestSolcCompilerError(t *testing.T) {
+
 	skipWithoutSolc(t)
 
-	contracts, err := CompileSolidityString("", testSource[4:])
+	solc, err := InitSolc("solc")
-	if err == nil {
+	if err != nil {
-		t.Errorf("error expected compiling source. got none. result %v", contracts)
+		t.Fatalf("Could not initialize solc: %v", err)
+	}
+	content := []byte(faultySol)
+	tmpfile, err := ioutil.TempFile("", "faultyContract.sol")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	err = writeToTempFile(tmpfile, content)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	flags := SolcFlagOpts{
+		CombinedOutput: []string{"bin", "abi"},
+	}
+
+	_, err = solc.Compile(FlagOpts{SolcFlagOpts: flags}, tmpfile.Name())
+
+	if err == nil {
+		t.Fatal("Expected an error, got nil.")
+	} else if !strings.Contains(err.Error(), "solc") {
+		t.Fatalf("Expected error to come directly from compiler, got err from elsewhere: %v", err)
+	}
+}
+
+func TestSolcCompilerWarning(t *testing.T) {
+
+	skipWithoutSolc(t)
+
+	solc, err := InitSolc("solc")
+	if err != nil {
+		t.Fatalf("Could not initialize solc: %v", err)
+	}
+	content := []byte(solNoPragma)
+	tmpfile, err := ioutil.TempFile("", "warningContract.sol")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	err = writeToTempFile(tmpfile, content)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	flags := SolcFlagOpts{
+		CombinedOutput: []string{"bin", "abi"},
+	}
+
+	solReturn, err := solc.Compile(FlagOpts{SolcFlagOpts: flags}, tmpfile.Name())
+	if err != nil {
+		t.Error(err)
+	}
+	if solReturn.Warning == "" {
+		t.Error("Expected a warning, got none.")
+	}
+}
+
+func TestLinkingBinaries(t *testing.T) {
+
+	skipWithoutSolc(t)
+
+	solc, err := InitSolc("solc")
+	if err != nil {
+		t.Fatalf("Could not initialize solc: %v", err)
+	}
+	content := []byte(simplyLibrarySol)
+	tmpfile, err := ioutil.TempFile("", "libraryContracts.sol")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	err = writeToTempFile(tmpfile, content)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	flags := SolcFlagOpts{
+		CombinedOutput: []string{"bin"},
+	}
+
+	solReturn, err := solc.Compile(FlagOpts{SolcFlagOpts: flags}, tmpfile.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if solReturn.Warning != "" || len(solReturn.Contracts) != 2 {
+		t.Fatalf("Expected no errors or warnings and expected contract items. Got %v for warnings, and %v for contract items", solReturn.Warning, solReturn.Contracts)
+	}
+
+	output, err := solc.(*Solidity).Link(map[string]common.Address{"Set": common.StringToAddress("0x692a70d2e424a56d2c6c27aa97d1a86395877b3a")}, solReturn.Contracts["C"].Bin)
+	if err != nil {
+		t.Error(err)
+	}
+	if strings.Contains(output, "_") {
+		t.Errorf("Expected binaries to link, but they did not")
+	}
+}
+
+func TestRemappings(t *testing.T) {
+
+	skipWithoutSolc(t)
+
+	solc, err := InitSolc("solc")
+	if err != nil {
+		t.Fatalf("Could not initialize solc: %v", err)
+	}
+
+	tmpfile1, err := ioutil.TempFile("", "C.sol")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile1.Name()) // clean up
+
+	err = writeToTempFile(tmpfile1, []byte(solFile1))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dir, err := ioutil.TempDir("", "tempDir")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer os.RemoveAll(dir) // clean up
+
+	tmpfn := filepath.Join(dir, "set.sol")
+	if err := ioutil.WriteFile(tmpfn, []byte(solFile2), 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	tmpfile2, err := ioutil.TempFile("", "main.sol")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(tmpfile2.Name()) // clean up
+
+	err = writeToTempFile(tmpfile2, []byte(regularSolFile))
+	if err != nil {
+		t.Fatal(err)
+	}
+	flags := SolcFlagOpts{
+		CombinedOutput: []string{"bin", "abi"},
+		Remappings:     []string{`/somedir/=` + dir + "/"},
+	}
+
+	solReturn, err := solc.Compile(FlagOpts{SolcFlagOpts: flags}, tmpfile1.Name(), tmpfile2.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if solReturn.Warning != "" || len(solReturn.Contracts) != 3 {
+		t.Fatalf("Expected no warnings and expected %v contract items. Got %v for warnings, and %v for contract items", 3, solReturn.Warning, len(solReturn.Contracts))
 	}
-	t.Logf("error: %v", err)
 }