v0.3.4

Add sleep tip about ssh

.travis.yml

@@ -5,7 +5,9 @@ go:
     - tip
 
 install: make deps
-script: make test
+script:
+    - go test ./...
+    - go test -race ./...
 
 notifications:
     flowdock:

CHANGELOG.md

@@ -1,4 +1,38 @@
-## 0.3.2 (unreleased)
+## 0.3.4 (August 21, 2013)
+
+IMPROVEMENTS:
+
+* post-processor/vagrant: the file being compressed will be shown
+  in the UI [GH-314]
+
+BUG FIXES:
+
+* core: Avoid panics when double-interrupting Packer.
+* provisioner/shell: Retry shell script uploads, making reboots more
+  robust if they happen to fail in this stage. [GH-282]
+
+## 0.3.3 (August 19, 2013)
+
+FEATURES:
+
+* builder/virtualbox: support exporting in OVA format. [GH-309]
+
+IMPROVEMENTS:
+
+* core: All HTTP downloads across Packer now support the standard
+  proxy environmental variables (`HTTP_PROXY`, `NO_PROXY`, etc.) [GH-252]
+* builder/amazon: API requests will use HTTP proxy if specified by
+  enviromental variables.
+* builder/digitalocean: API requests will use HTTP proxy if specified
+  by environmental variables.
+
+BUG FIXES:
+
+* core: TCP connection between plugin processes will keep-alive. [GH-312]
+* core: No more "unused key keep_input_artifact" for post processors [GH-310]
+* post-processor/vagrant: `output_path` templates now work again.
+
+## 0.3.2 (August 18, 2013)
 
 FEATURES:
 

builder/digitalocean/api.go

@@ -43,7 +43,11 @@ type DigitalOceanClient struct {
 // Creates a new client for communicating with DO
 func (d DigitalOceanClient) New(client string, key string) *DigitalOceanClient {
 	c := &DigitalOceanClient{
-		client:   http.DefaultClient,
+		client: &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+			},
+		},
 		BaseURL:  DIGITALOCEAN_API_URL,
 		ClientID: client,
 		APIKey:   key,

builder/virtualbox/builder.go

@@ -27,6 +27,7 @@ type config struct {
 	BootCommand          []string   `mapstructure:"boot_command"`
 	DiskSize             uint       `mapstructure:"disk_size"`
 	FloppyFiles          []string   `mapstructure:"floppy_files"`
+	Format               string     `mapstructure:"format"`
 	GuestAdditionsPath   string     `mapstructure:"guest_additions_path"`
 	GuestAdditionsURL    string     `mapstructure:"guest_additions_url"`
 	GuestAdditionsSHA256 string     `mapstructure:"guest_additions_sha256"`
@@ -131,6 +132,10 @@ func (b *Builder) Prepare(raws ...interface{}) error {
 		b.config.VMName = fmt.Sprintf("packer-%s", b.config.PackerBuildName)
 	}
 
+	if b.config.Format == "" {
+		b.config.Format = "ovf"
+	}
+
 	// Errors
 	templates := map[string]*string{
 		"guest_additions_sha256":  &b.config.GuestAdditionsSHA256,
@@ -145,6 +150,7 @@ func (b *Builder) Prepare(raws ...interface{}) error {
 		"ssh_username":            &b.config.SSHUser,
 		"virtualbox_version_file": &b.config.VBoxVersionFile,
 		"vm_name":                 &b.config.VMName,
+		"format":                  &b.config.Format,
 		"boot_wait":               &b.config.RawBootWait,
 		"shutdown_timeout":        &b.config.RawShutdownTimeout,
 		"ssh_wait_timeout":        &b.config.RawSSHWaitTimeout,
@@ -197,6 +203,11 @@ func (b *Builder) Prepare(raws ...interface{}) error {
 		}
 	}
 
+	if !(b.config.Format == "ovf" || b.config.Format == "ova") {
+		errs = packer.MultiErrorAppend(
+			errs, errors.New("invalid format, only 'ovf' or 'ova' are allowed"))
+	}
+
 	if b.config.HTTPPortMin > b.config.HTTPPortMax {
 		errs = packer.MultiErrorAppend(
 			errs, errors.New("http_port_min must be less than http_port_max"))

builder/virtualbox/builder_test.go

@@ -58,6 +58,10 @@ func TestBuilderPrepare_Defaults(t *testing.T) {
 	if b.config.VMName != "packer-foo" {
 		t.Errorf("bad vm name: %s", b.config.VMName)
 	}
+
+	if b.config.Format != "ovf" {
+		t.Errorf("bad format: %s", b.config.Format)
+	}
 }
 
 func TestBuilderPrepare_BootWait(t *testing.T) {
@@ -248,6 +252,34 @@ func TestBuilderPrepare_HTTPPort(t *testing.T) {
 	}
 }
 
+func TestBuilderPrepare_Format(t *testing.T) {
+	var b Builder
+	config := testConfig()
+
+	// Bad
+	config["format"] = "illegal value"
+	err := b.Prepare(config)
+	if err == nil {
+		t.Fatal("should have error")
+	}
+
+	// Good
+	config["format"] = "ova"
+	b = Builder{}
+	err = b.Prepare(config)
+	if err != nil {
+		t.Fatalf("should not have error: %s", err)
+	}
+
+	// Good
+	config["format"] = "ovf"
+	b = Builder{}
+	err = b.Prepare(config)
+	if err != nil {
+		t.Fatalf("should not have error: %s", err)
+	}
+}
+
 func TestBuilderPrepare_InvalidKey(t *testing.T) {
 	var b Builder
 	config := testConfig()

builder/virtualbox/step_export.go

@@ -50,7 +50,7 @@ func (s *stepExport) Run(state map[string]interface{}) multistep.StepAction {
 	}
 
 	// Export the VM to an OVF
-	outputPath := filepath.Join(config.OutputDir, "packer.ovf")
+	outputPath := filepath.Join(config.OutputDir, "packer."+config.Format)
 
 	command = []string{
 		"export",

common/download.go

@@ -188,7 +188,18 @@ func (*HTTPDownloader) Cancel() {
 
 func (d *HTTPDownloader) Download(dst io.Writer, src *url.URL) error {
 	log.Printf("Starting download: %s", src.String())
-	resp, err := http.Get(src.String())
+	req, err := http.NewRequest("GET", src.String(), nil)
+	if err != nil {
+		return err
+	}
+
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+		},
+	}
+
+	resp, err := httpClient.Do(req)
 	if err != nil {
 		return err
 	}

communicator/ssh/communicator_test.go

@@ -1,3 +1,5 @@
+// +build !race
+
 package ssh
 
 import (

packer/build.go

@@ -90,7 +90,7 @@ type coreBuild struct {
 type coreBuildPostProcessor struct {
 	processor         PostProcessor
 	processorType     string
-	config            interface{}
+	config            map[string]interface{}
 	keepInputArtifact bool
 }
 

packer/build_test.go

@@ -20,7 +20,7 @@ func testBuild() *coreBuild {
 		},
 		postProcessors: [][]coreBuildPostProcessor{
 			[]coreBuildPostProcessor{
-				coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp"}, "testPP", 42, true},
+				coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp"}, "testPP", make(map[string]interface{}), true},
 			},
 		},
 		variables: make(map[string]string),
@@ -66,7 +66,7 @@ func TestBuild_Prepare(t *testing.T) {
 	corePP := build.postProcessors[0][0]
 	pp := corePP.processor.(*TestPostProcessor)
 	assert.True(pp.configCalled, "config should be called")
-	assert.Equal(pp.configVal, []interface{}{42, packerConfig}, "config should have right value")
+	assert.Equal(pp.configVal, []interface{}{make(map[string]interface{}), packerConfig}, "config should have right value")
 }
 
 func TestBuild_Prepare_Twice(t *testing.T) {
@@ -231,7 +231,7 @@ func TestBuild_Run_Artifacts(t *testing.T) {
 	build = testBuild()
 	build.postProcessors = [][]coreBuildPostProcessor{
 		[]coreBuildPostProcessor{
-			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp"}, "pp", 42, false},
+			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp"}, "pp", make(map[string]interface{}), false},
 		},
 	}
 
@@ -256,10 +256,10 @@ func TestBuild_Run_Artifacts(t *testing.T) {
 	build = testBuild()
 	build.postProcessors = [][]coreBuildPostProcessor{
 		[]coreBuildPostProcessor{
-			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp1"}, "pp", 42, false},
+			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp1"}, "pp", make(map[string]interface{}), false},
 		},
 		[]coreBuildPostProcessor{
-			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp2"}, "pp", 42, true},
+			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp2"}, "pp", make(map[string]interface{}), true},
 		},
 	}
 
@@ -284,12 +284,12 @@ func TestBuild_Run_Artifacts(t *testing.T) {
 	build = testBuild()
 	build.postProcessors = [][]coreBuildPostProcessor{
 		[]coreBuildPostProcessor{
-			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp1a"}, "pp", 42, false},
-			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp1b"}, "pp", 42, true},
+			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp1a"}, "pp", make(map[string]interface{}), false},
+			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp1b"}, "pp", make(map[string]interface{}), true},
 		},
 		[]coreBuildPostProcessor{
-			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp2a"}, "pp", 42, false},
-			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp2b"}, "pp", 42, false},
+			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp2a"}, "pp", make(map[string]interface{}), false},
+			coreBuildPostProcessor{&TestPostProcessor{artifactId: "pp2b"}, "pp", make(map[string]interface{}), false},
 		},
 	}
 
@@ -315,7 +315,7 @@ func TestBuild_Run_Artifacts(t *testing.T) {
 	build.postProcessors = [][]coreBuildPostProcessor{
 		[]coreBuildPostProcessor{
 			coreBuildPostProcessor{
-				&TestPostProcessor{artifactId: "pp", keep: true}, "pp", 42, false,
+				&TestPostProcessor{artifactId: "pp", keep: true}, "pp", make(map[string]interface{}), false,
 			},
 		},
 	}

packer/communicator.go

@@ -33,9 +33,11 @@ type RemoteCmd struct {
 	// Once Exited is true, this will contain the exit code of the process.
 	ExitStatus int
 
-	// Internal locks and such used for safely setting some shared variables
-	l      sync.Mutex
+	// Internal fields
 	exitCh chan struct{}
+
+	// This thing is a mutex, lock when making modifications concurrently
+	sync.Mutex
 }
 
 // A Communicator is the interface used to communicate with the machine
@@ -76,6 +78,9 @@ func (r *RemoteCmd) StartWithUi(c Communicator, ui Ui) error {
 	originalStdout := r.Stdout
 	originalStderr := r.Stderr
 	defer func() {
+		r.Lock()
+		defer r.Unlock()
+
 		r.Stdout = originalStdout
 		r.Stderr = originalStderr
 	}()
@@ -141,8 +146,8 @@ OutputLoop:
 // should be called by communicators who are running a remote command in
 // order to set that the command is done.
 func (r *RemoteCmd) SetExited(status int) {
-	r.l.Lock()
-	defer r.l.Unlock()
+	r.Lock()
+	defer r.Unlock()
 
 	if r.exitCh == nil {
 		r.exitCh = make(chan struct{})
@@ -156,11 +161,11 @@ func (r *RemoteCmd) SetExited(status int) {
 // Wait waits for the remote command to complete.
 func (r *RemoteCmd) Wait() {
 	// Make sure our condition variable is initialized.
-	r.l.Lock()
+	r.Lock()
 	if r.exitCh == nil {
 		r.exitCh = make(chan struct{})
 	}
-	r.l.Unlock()
+	r.Unlock()
 
 	<-r.exitCh
 }

packer/communicator_test.go

@@ -15,6 +15,9 @@ type TestCommunicator struct {
 
 func (c *TestCommunicator) Start(rc *RemoteCmd) error {
 	go func() {
+		rc.Lock()
+		defer rc.Unlock()
+
 		if rc.Stdout != nil && c.Stdout != nil {
 			io.Copy(rc.Stdout, c.Stdout)
 		}

packer/plugin/builder.go

@@ -40,7 +40,7 @@ func (b *cmdBuilder) Cancel() {
 func (c *cmdBuilder) checkExit(p interface{}, cb func()) {
 	if c.client.Exited() && cb != nil {
 		cb()
-	} else if p != nil {
+	} else if p != nil && !Killed {
 		log.Panic(p)
 	}
 }

packer/plugin/client.go

@@ -2,7 +2,6 @@ package plugin
 
 import (
 	"bufio"
-	"bytes"
 	"errors"
 	"fmt"
 	"github.com/mitchellh/packer/packer"
@@ -10,6 +9,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/rpc"
 	"os"
 	"os/exec"
@@ -19,6 +19,10 @@ import (
 	"unicode"
 )
 
+// If this is true, then the "unexpected EOF" panic will not be
+// raised throughout the clients.
+var Killed = false
+
 // This is a slice of the "managed" clients which are cleaned up when
 // calling Cleanup
 var managedClients = make([]*Client, 0, 5)
@@ -68,6 +72,9 @@ type ClientConfig struct {
 //
 // This must only be called _once_.
 func CleanupClients() {
+	// Set the killed to true so that we don't get unexpected panics
+	Killed = true
+
 	// Kill all the managed clients in parallel and use a WaitGroup
 	// to wait for them all to finish up.
 	var wg sync.WaitGroup
@@ -115,6 +122,8 @@ func NewClient(config *ClientConfig) (c *Client) {
 
 // Tells whether or not the underlying process has exited.
 func (c *Client) Exited() bool {
+	c.l.Lock()
+	defer c.l.Unlock()
 	return c.exited
 }
 
@@ -214,7 +223,7 @@ func (c *Client) Start() (address string, err error) {
 		fmt.Sprintf("PACKER_PLUGIN_MAX_PORT=%d", c.config.MaxPort),
 	}
 
-	stdout := new(bytes.Buffer)
+	stdout_r, stdout_w := io.Pipe()
 	stderr_r, stderr_w := io.Pipe()
 
 	cmd := c.config.Cmd
@@ -222,7 +231,7 @@ func (c *Client) Start() (address string, err error) {
 	cmd.Env = append(cmd.Env, env...)
 	cmd.Stdin = os.Stdin
 	cmd.Stderr = stderr_w
-	cmd.Stdout = stdout
+	cmd.Stdout = stdout_w
 
 	log.Printf("Starting plugin: %s %#v", cmd.Path, cmd.Args)
 	err = cmd.Start()
@@ -244,10 +253,12 @@ func (c *Client) Start() (address string, err error) {
 	}()
 
 	// Start goroutine to wait for process to exit
+	exitCh := make(chan struct{})
 	go func() {
-		// Make sure we close the write end of our stderr listener so
-		// that the log goroutine ends properly.
+		// Make sure we close the write end of our stderr/stdout so
+		// that the readers send EOF properly.
 		defer stderr_w.Close()
+		defer stdout_w.Close()
 
 		// Wait for the command to end.
 		cmd.Wait()
@@ -257,46 +268,60 @@ func (c *Client) Start() (address string, err error) {
 		os.Stderr.Sync()
 
 		// Mark that we exited
+		close(exitCh)
+
+		// Set that we exited, which takes a lock
+		c.l.Lock()
+		defer c.l.Unlock()
 		c.exited = true
 	}()
 
 	// Start goroutine that logs the stderr
 	go c.logStderr(stderr_r)
 
+	// Start a goroutine that is going to be reading the lines
+	// out of stdout
+	linesCh := make(chan []byte)
+	go func() {
+		defer close(linesCh)
+
+		buf := bufio.NewReader(stdout_r)
+		for {
+			line, err := buf.ReadBytes('\n')
+			if line != nil {
+				linesCh <- line
+			}
+
+			if err == io.EOF {
+				return
+			}
+		}
+	}()
+
+	// Make sure after we exit we read the lines from stdout forever
+	// so they dont' block since it is an io.Pipe
+	defer func() {
+		go func() {
+			for _ = range linesCh {
+			}
+		}()
+	}()
+
 	// Some channels for the next step
 	timeout := time.After(c.config.StartTimeout)
 
 	// Start looking for the address
 	log.Printf("Waiting for RPC address for: %s", cmd.Path)
-	for done := false; !done; {
-		select {
-		case <-timeout:
-			err = errors.New("timeout while waiting for plugin to start")
-			done = true
-		default:
-		}
-
-		if err == nil && c.Exited() {
-			err = errors.New("plugin exited before we could connect")
-			done = true
-		}
-
-		if line, lerr := stdout.ReadBytes('\n'); lerr == nil {
-			// Trim the address and reset the err since we were able
-			// to read some sort of address.
-			c.address = strings.TrimSpace(string(line))
-			address = c.address
-			err = nil
-			break
-		}
-
-		// If error is nil from previously, return now
-		if err != nil {
-			return
-		}
-
-		// Wait a bit
-		time.Sleep(10 * time.Millisecond)
+	select {
+	case <-timeout:
+		err = errors.New("timeout while waiting for plugin to start")
+	case <-exitCh:
+		err = errors.New("plugin exited before we could connect")
+	case line := <-linesCh:
+		// Trim the address and reset the err since we were able
+		// to read some sort of address.
+		c.address = strings.TrimSpace(string(line))
+		address = c.address
 	}
 
 	return
@@ -328,10 +353,14 @@ func (c *Client) rpcClient() (*rpc.Client, error) {
 		return nil, err
 	}
 
-	client, err := rpc.Dial("tcp", address)
+	conn, err := net.Dial("tcp", address)
 	if err != nil {
 		return nil, err
 	}
 
-	return client, nil
+	// Make sure to set keep alive so that the connection doesn't die
+	tcpConn := conn.(*net.TCPConn)
+	tcpConn.SetKeepAlive(true)
+
+	return rpc.NewClient(tcpConn), nil
 }

packer/plugin/command.go

@@ -45,7 +45,7 @@ func (c *cmdCommand) Synopsis() (result string) {
 func (c *cmdCommand) checkExit(p interface{}, cb func()) {
 	if c.client.Exited() {
 		cb()
-	} else if p != nil {
+	} else if p != nil && !Killed {
 		log.Panic(p)
 	}
 }

packer/plugin/hook.go

@@ -22,7 +22,7 @@ func (c *cmdHook) Run(name string, ui packer.Ui, comm packer.Communicator, data
 func (c *cmdHook) checkExit(p interface{}, cb func()) {
 	if c.client.Exited() {
 		cb()
-	} else if p != nil {
+	} else if p != nil && !Killed {
 		log.Panic(p)
 	}
 }

packer/plugin/post_processor.go

@@ -31,7 +31,7 @@ func (c *cmdPostProcessor) PostProcess(ui packer.Ui, a packer.Artifact) (packer.
 func (c *cmdPostProcessor) checkExit(p interface{}, cb func()) {
 	if c.client.Exited() {
 		cb()
-	} else if p != nil {
+	} else if p != nil && !Killed {
 		log.Panic(p)
 	}
 }

packer/plugin/provisioner.go

@@ -31,7 +31,7 @@ func (c *cmdProvisioner) Provision(ui packer.Ui, comm packer.Communicator) error
 func (c *cmdProvisioner) checkExit(p interface{}, cb func()) {
 	if c.client.Exited() {
 		cb()
-	} else if p != nil {
+	} else if p != nil && !Killed {
 		log.Panic(p)
 	}
 }

packer/rpc/communicator.go

@@ -78,7 +78,8 @@ func (c *communicator) Start(cmd *packer.RemoteCmd) (err error) {
 
 		conn, err := responseL.Accept()
 		if err != nil {
-			log.Panic(err)
+			cmd.SetExited(123)
+			return
 		}
 
 		defer conn.Close()
@@ -87,7 +88,8 @@ func (c *communicator) Start(cmd *packer.RemoteCmd) (err error) {
 
 		var finished CommandFinished
 		if err := decoder.Decode(&finished); err != nil {
-			log.Panic(err)
+			cmd.SetExited(123)
+			return
 		}
 
 		cmd.SetExited(finished.ExitStatus)

packer/rpc/communicator_test.go

@@ -99,7 +99,10 @@ func TestCommunicatorRPC(t *testing.T) {
 	c.startCmd.SetExited(42)
 
 	for i := 0; i < 5; i++ {
-		if cmd.Exited {
+		cmd.Lock()
+		exited := cmd.Exited
+		cmd.Unlock()
+		if exited {
 			assert.Equal(cmd.ExitStatus, 42, "should have proper exit status")
 			break
 		}

packer/template.go

@@ -50,7 +50,7 @@ type RawBuilderConfig struct {
 type RawPostProcessorConfig struct {
 	Type              string
 	KeepInputArtifact bool `mapstructure:"keep_input_artifact"`
-	RawConfig         interface{}
+	RawConfig         map[string]interface{}
 }
 
 // RawProvisionerConfig represents a raw, unprocessed provisioner configuration.
@@ -162,7 +162,7 @@ func ParseTemplate(data []byte) (t *Template, err error) {
 	// are actually three different formats that the user can use to define
 	// a post-processor.
 	for i, rawV := range rawTpl.PostProcessors {
-		rawPP, err := parsePostProvisioner(i, rawV)
+		rawPP, err := parsePostProcessor(i, rawV)
 		if err != nil {
 			errors = append(errors, err...)
 			continue
@@ -189,6 +189,9 @@ func ParseTemplate(data []byte) (t *Template, err error) {
 				continue
 			}
 
+			// Remove the input keep_input_artifact option
+			delete(pp, "keep_input_artifact")
+
 			config.RawConfig = pp
 		}
 	}
@@ -260,7 +263,7 @@ func ParseTemplateFile(path string) (*Template, error) {
 	return ParseTemplate(data)
 }
 
-func parsePostProvisioner(i int, rawV interface{}) (result []map[string]interface{}, errors []error) {
+func parsePostProcessor(i int, rawV interface{}) (result []map[string]interface{}, errors []error) {
 	switch v := rawV.(type) {
 	case string:
 		result = []map[string]interface{}{

packer/template_test.go

@@ -623,6 +623,11 @@ func TestTemplate_Build(t *testing.T) {
 	assert.Equal(len(coreBuild.postProcessors[1]), 2, "should have correct number")
 	assert.False(coreBuild.postProcessors[1][0].keepInputArtifact, "shoule be correct")
 	assert.True(coreBuild.postProcessors[1][1].keepInputArtifact, "shoule be correct")
+
+	config := coreBuild.postProcessors[1][1].config
+	if _, ok := config["keep_input_artifact"]; ok {
+		t.Fatal("should not have keep_input_artifact")
+	}
 }
 
 func TestTemplate_Build_ProvisionerOverride(t *testing.T) {

packer/version.go

@@ -10,12 +10,12 @@ import (
 var GitCommit string
 
 // The version of packer.
-const Version = "0.3.2"
+const Version = "0.3.4"
 
 // Any pre-release marker for the version. If this is "" (empty string),
 // then it means that it is a final release. Otherwise, this is the
 // pre-release marker.
-const VersionPrerelease = "dev"
+const VersionPrerelease = ""
 
 type versionCommand byte
 

post-processor/vagrant/aws.go

@@ -43,20 +43,8 @@ func (p *AWSBoxPostProcessor) Configure(raws ...interface{}) error {
 	// Accumulate any errors
 	errs := common.CheckUnusedConfig(md)
 
-	templates := map[string]*string{
-		"output": &p.config.OutputPath,
-	}
-
-	for n, ptr := range templates {
-		var err error
-		*ptr, err = p.config.tpl.Process(*ptr, nil)
-		if err != nil {
-			errs = packer.MultiErrorAppend(
-				errs, fmt.Errorf("Error processing %s: %s", n, err))
-		}
-	}
-
 	validates := map[string]*string{
+		"output":               &p.config.OutputPath,
 		"vagrantfile_template": &p.config.VagrantfileTemplate,
 	}
 
@@ -90,8 +78,11 @@ func (p *AWSBoxPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact
 	}
 
 	// Compile the output path
-	outputPath, err := ProcessOutputPath(p.config.OutputPath,
-		p.config.PackerBuildName, "aws", artifact)
+	outputPath, err := p.config.tpl.Process(p.config.OutputPath, &OutputPathTemplate{
+		ArtifactId: artifact.Id(),
+		BuildName:  p.config.PackerBuildName,
+		Provider:   "aws",
+	})
 	if err != nil {
 		return nil, false, err
 	}
@@ -143,7 +134,7 @@ func (p *AWSBoxPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact
 	}
 
 	// Compress the directory to the given output path
-	if err := DirToBox(outputPath, dir); err != nil {
+	if err := DirToBox(outputPath, dir, ui); err != nil {
 		err = fmt.Errorf("error creating box: %s", err)
 		return nil, false, err
 	}

post-processor/vagrant/util.go

@@ -2,15 +2,14 @@ package vagrant
 
 import (
 	"archive/tar"
-	"bytes"
 	"compress/gzip"
 	"encoding/json"
+	"fmt"
 	"github.com/mitchellh/packer/packer"
 	"io"
 	"log"
 	"os"
 	"path/filepath"
-	"text/template"
 )
 
 // OutputPathTemplate is the structure that is availalable within the
@@ -45,7 +44,7 @@ func CopyContents(dst, src string) error {
 // DirToBox takes the directory and compresses it into a Vagrant-compatible
 // box. This function does not perform checks to verify that dir is
 // actually a proper box. This is an expected precondition.
-func DirToBox(dst, dir string) error {
+func DirToBox(dst, dir string, ui packer.Ui) error {
 	log.Printf("Turning dir into box: %s => %s", dir, dst)
 	dstF, err := os.Create(dst)
 	if err != nil {
@@ -93,6 +92,10 @@ func DirToBox(dst, dir string) error {
 			return err
 		}
 
+		if ui != nil {
+			ui.Message(fmt.Sprintf("Compressing: %s", header.Name))
+		}
+
 		if err := tarWriter.WriteHeader(header); err != nil {
 			return err
 		}
@@ -108,26 +111,6 @@ func DirToBox(dst, dir string) error {
 	return filepath.Walk(dir, tarWalk)
 }
 
-// ProcessOutputPath takes an output path template and executes it,
-// replacing variables with their respective values.
-func ProcessOutputPath(path string, buildName string, provider string, artifact packer.Artifact) (string, error) {
-	var buf bytes.Buffer
-
-	tplData := &OutputPathTemplate{
-		ArtifactId: artifact.Id(),
-		BuildName:  buildName,
-		Provider:   provider,
-	}
-
-	t, err := template.New("output").Parse(path)
-	if err != nil {
-		return "", err
-	}
-
-	err = t.Execute(&buf, tplData)
-	return buf.String(), err
-}
-
 // WriteMetadata writes the "metadata.json" file for a Vagrant box.
 func WriteMetadata(dir string, contents interface{}) error {
 	f, err := os.Create(filepath.Join(dir, "metadata.json"))

post-processor/vagrant/virtualbox.go

@@ -45,20 +45,8 @@ func (p *VBoxBoxPostProcessor) Configure(raws ...interface{}) error {
 	// Accumulate any errors
 	errs := common.CheckUnusedConfig(md)
 
-	templates := map[string]*string{
-		"output": &p.config.OutputPath,
-	}
-
-	for n, ptr := range templates {
-		var err error
-		*ptr, err = p.config.tpl.Process(*ptr, nil)
-		if err != nil {
-			errs = packer.MultiErrorAppend(
-				errs, fmt.Errorf("Error processing %s: %s", n, err))
-		}
-	}
-
 	validates := map[string]*string{
+		"output":               &p.config.OutputPath,
 		"vagrantfile_template": &p.config.VagrantfileTemplate,
 	}
 
@@ -85,8 +73,11 @@ func (p *VBoxBoxPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifac
 	}
 
 	// Compile the output path
-	outputPath, err := ProcessOutputPath(p.config.OutputPath,
-		p.config.PackerBuildName, "virtualbox", artifact)
+	outputPath, err := p.config.tpl.Process(p.config.OutputPath, &OutputPathTemplate{
+		ArtifactId: artifact.Id(),
+		BuildName:  p.config.PackerBuildName,
+		Provider:   "virtualbox",
+	})
 	if err != nil {
 		return nil, false, err
 	}
@@ -152,7 +143,7 @@ func (p *VBoxBoxPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifac
 
 	// Compress the directory to the given output path
 	ui.Message(fmt.Sprintf("Compressing box..."))
-	if err := DirToBox(outputPath, dir); err != nil {
+	if err := DirToBox(outputPath, dir, ui); err != nil {
 		return nil, false, err
 	}
 

post-processor/vagrant/vmware.go

@@ -37,20 +37,8 @@ func (p *VMwareBoxPostProcessor) Configure(raws ...interface{}) error {
 	// Accumulate any errors
 	errs := common.CheckUnusedConfig(md)
 
-	templates := map[string]*string{
-		"output": &p.config.OutputPath,
-	}
-
-	for n, ptr := range templates {
-		var err error
-		*ptr, err = p.config.tpl.Process(*ptr, nil)
-		if err != nil {
-			errs = packer.MultiErrorAppend(
-				errs, fmt.Errorf("Error processing %s: %s", n, err))
-		}
-	}
-
 	validates := map[string]*string{
+		"output":               &p.config.OutputPath,
 		"vagrantfile_template": &p.config.VagrantfileTemplate,
 	}
 
@@ -70,8 +58,11 @@ func (p *VMwareBoxPostProcessor) Configure(raws ...interface{}) error {
 
 func (p *VMwareBoxPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
 	// Compile the output path
-	outputPath, err := ProcessOutputPath(p.config.OutputPath,
-		p.config.PackerBuildName, "vmware", artifact)
+	outputPath, err := p.config.tpl.Process(p.config.OutputPath, &OutputPathTemplate{
+		ArtifactId: artifact.Id(),
+		BuildName:  p.config.PackerBuildName,
+		Provider:   "vmware",
+	})
 	if err != nil {
 		return nil, false, err
 	}
@@ -128,7 +119,7 @@ func (p *VMwareBoxPostProcessor) PostProcess(ui packer.Ui, artifact packer.Artif
 
 	// Compress the directory to the given output path
 	ui.Message(fmt.Sprintf("Compressing box..."))
-	if err := DirToBox(outputPath, dir); err != nil {
+	if err := DirToBox(outputPath, dir, ui); err != nil {
 		return nil, false, err
 	}
 

provisioner/shell/provisioner.go

@@ -237,7 +237,9 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
 		defer f.Close()
 
 		log.Printf("Uploading %s => %s", path, p.config.RemotePath)
-		err = comm.Upload(p.config.RemotePath, f)
+		err = p.retryable(func() error {
+			return comm.Upload(p.config.RemotePath, f)
+		})
 		if err != nil {
 			return fmt.Errorf("Error uploading shell script: %s", err)
 		}
@@ -258,26 +260,13 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
 		}
 
 		cmd := &packer.RemoteCmd{Command: command}
-		startTimeout := time.After(p.config.startRetryTimeout)
 		log.Printf("Executing command: %s", cmd.Command)
-		for {
-			if err := cmd.StartWithUi(comm, ui); err == nil {
-				break
-			}
+		err = p.retryable(func() error {
+			return cmd.StartWithUi(comm, ui)
+		})
 
-			// Create an error and log it
-			err = fmt.Errorf("Error executing command: %s", err)
-			log.Printf(err.Error())
-
-			// Check if we timed out, otherwise we retry. It is safe to
-			// retry since the only error case above is if the command
-			// failed to START.
-			select {
-			case <-startTimeout:
-				return err
-			default:
-				time.Sleep(2 * time.Second)
-			}
+		if err != nil {
+			return err
 		}
 
 		if cmd.ExitStatus != 0 {
@@ -287,3 +276,29 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
 
 	return nil
 }
+
+// retryable will retry the given function over and over until a
+// non-error is returned.
+func (p *Provisioner) retryable(f func() error) error {
+	startTimeout := time.After(p.config.startRetryTimeout)
+	for {
+		var err error
+		if err = f(); err == nil {
+			return nil
+		}
+
+		// Create an error and log it
+		err = fmt.Errorf("Retryable error: %s", err)
+		log.Printf(err.Error())
+
+		// Check if we timed out, otherwise we retry. It is safe to
+		// retry since the only error case above is if the command
+		// failed to START.
+		select {
+		case <-startTimeout:
+			return err
+		default:
+			time.Sleep(2 * time.Second)
+		}
+	}
+}

scripts/build.sh

@@ -20,9 +20,19 @@ cd $DIR
 GIT_COMMIT=$(git rev-parse HEAD)
 GIT_DIRTY=$(test -n "`git status --porcelain`" && echo "+CHANGES" || true)
 
+# If we're building a race-enabled build, then set that up.
+if [ ! -z $PACKER_RACE ]; then
+    echo -e "${OK_COLOR}--> Building with race detection enabled${NO_COLOR}"
+    PACKER_RACE="-race"
+fi
+
+echo -e "${OK_COLOR}--> Installing dependencies to speed up builds...${NO_COLOR}"
+go get ./...
+
 # Compile the main Packer app
 echo -e "${OK_COLOR}--> Compiling Packer${NO_COLOR}"
 go build \
+    ${PACKER_RACE} \
     -ldflags "-X github.com/mitchellh/packer/packer.GitCommit ${GIT_COMMIT}${GIT_DIRTY}" \
     -v \
     -o bin/packer .
@@ -32,6 +42,7 @@ for PLUGIN in $(find ./plugin -mindepth 1 -maxdepth 1 -type d); do
     PLUGIN_NAME=$(basename ${PLUGIN})
     echo -e "${OK_COLOR}--> Compiling Plugin: ${PLUGIN_NAME}${NO_COLOR}"
     go build \
+        ${PACKER_RACE} \
         -ldflags "-X github.com/mitchellh/packer/packer.GitCommit ${GIT_COMMIT}${GIT_DIRTY}" \
         -v \
         -o bin/packer-${PLUGIN_NAME} ${PLUGIN}

scripts/dist.sh

@@ -60,15 +60,11 @@ waitAll() {
 trap "kill 0" SIGINT SIGTERM EXIT
 
 # Build our root project
-xc &
+xc
 
 # Build all the plugins
 for PLUGIN in $(find ./plugin -mindepth 1 -maxdepth 1 -type d); do
     PLUGIN_NAME=$(basename ${PLUGIN})
-    (
-    pushd ${PLUGIN}
-    xc
-    popd
     find ./pkg \
         -type f \
         -name ${PLUGIN_NAME} \
@@ -77,11 +73,8 @@ for PLUGIN in $(find ./plugin -mindepth 1 -maxdepth 1 -type d); do
         -type f \
         -name ${PLUGIN_NAME}.exe \
         -execdir mv ${PLUGIN_NAME}.exe packer-${PLUGIN_NAME}.exe ';'
-    ) &
 done
 
-waitAll
-
 # Zip all the packages
 mkdir -p ./pkg/${VERSIONDIR}/dist
 for PLATFORM in $(find ./pkg/${VERSIONDIR} -mindepth 1 -maxdepth 1 -type d); do

signal.go

@@ -26,7 +26,8 @@ func setupSignalHandlers(env packer.Environment) {
 
 		env.Ui().Error("Interrupt signal received twice. Forcefully exiting now.")
 
-		// Force kill all the plugins
+		// Force kill all the plugins, but mark that we're killing them
+		// first so that we don't get panics everywhere.
 		plugin.CleanupClients()
 		os.Exit(1)
 	}()

website/config.rb

@@ -45,7 +45,9 @@ set :images_dir, 'images'
 
 # Use the RedCarpet Markdown engine
 set :markdown_engine, :redcarpet
-set :markdown, :fenced_code_blocks => true
+set :markdown,
+    :fenced_code_blocks => true,
+    :with_toc_data => true
 
 # Build-specific configuration
 configure :build do

website/source/docs/builders/virtualbox.html.markdown

@@ -82,6 +82,9 @@ Optional:
   be attached. The files listed in this configuration will all be put
   into the root directory of the floppy disk; sub-directories are not supported.
 
+* `format` (string) - Either "ovf" or "ova", this specifies the output
+  format of the exported virtual machine. This defaults to "ovf".
+
 * `guest_additions_path` (string) - The path on the guest virtual machine
   where the VirtualBox guest additions ISO will be uploaded. By default this
   is "VBoxGuestAdditions.iso" which should upload into the login directory

website/source/docs/extend/builder.html.markdown

@@ -1,5 +1,6 @@
 ---
 layout: "docs"
+page_title: "Custom Builder - Extend Packer"
 ---
 
 # Custom Builder Development

website/source/docs/extend/plugins.html.markdown

@@ -1,5 +1,6 @@
 ---
 layout: "docs"
+page_title: "Packer Plugins - Extend Packer"
 ---
 
 # Packer Plugins

website/source/docs/provisioners/shell.html.markdown

@@ -127,3 +127,36 @@ reboot so that SSH will eventually be killed automatically:
 reboot
 sleep 60
 ```
+
+## Troubleshooting
+
+*My shell script doesn't work correctly on Ubuntu*
+
+* On Ubuntu the /bin/sh shell is
+[dash](http://en.wikipedia.org/wiki/Debian_Almquist_shell). If your script has
+[bash](http://en.wikipedia.org/wiki/Bash_(Unix_shell\)) specific commands in it
+then put `#!/bin/bash` at the top of your script. Differences
+between dash and bash can be found on the [DashAsBinSh](https://wiki.ubuntu.com/DashAsBinSh) Ubuntu wiki page.
+
+*My shell works when I login but fails with the shell provisioner*
+
+* See the above tip. More than likely your login shell is using /bin/bash
+while the provisioner is using /bin/sh.
+
+*How do I tell what my shell script is doing?*
+
+* Adding a `-x` flag to the shebang at the top of the script (`#!/bin/sh -x`)
+will echo the script statements as it is executing.
+
+*My builds don't always work the same*
+
+* Some distributions start the SSH daemon before other core services which
+can create race conditions. Your first provisoner can tell the machine to
+wait until it completely boots.
+
+<pre class="prettyprint">
+    {
+       "type": "script",
+       "inline": [ "sleep 10" ]
+    }
+</pre>