From e9246ec490a50a5e67fc56ed5e4092c30282b9ac Mon Sep 17 00:00:00 2001 From: Christian Groschupp Date: Sun, 16 Nov 2014 19:31:08 +0100 Subject: [PATCH 01/86] builder/vmware/iso: Upload VMX to ESX5 after editing [GH-1422,GH-1083] --- builder/vmware/iso/builder.go | 3 ++ builder/vmware/iso/driver_esx5.go | 4 ++ builder/vmware/iso/remote_driver.go | 6 +++ builder/vmware/iso/remote_driver_mock.go | 12 ++++++ builder/vmware/iso/step_upload_vmx.go | 48 ++++++++++++++++++++++++ 5 files changed, 73 insertions(+) create mode 100644 builder/vmware/iso/step_upload_vmx.go diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go index aeda7ff41..674b308c9 100644 --- a/builder/vmware/iso/builder.go +++ b/builder/vmware/iso/builder.go @@ -368,6 +368,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SkipFloppy: true, }, &vmwcommon.StepCleanVMX{}, + &StepUploadVMX{ + RemoteType: b.config.RemoteType, + }, &vmwcommon.StepCompactDisk{ Skip: b.config.SkipCompaction, }, diff --git a/builder/vmware/iso/driver_esx5.go b/builder/vmware/iso/driver_esx5.go index ca1733d63..1f9bd7a78 100644 --- a/builder/vmware/iso/driver_esx5.go +++ b/builder/vmware/iso/driver_esx5.go @@ -56,6 +56,10 @@ func (d *ESX5Driver) IsRunning(string) (bool, error) { return strings.Contains(state, "Powered on"), nil } +func (d *ESX5Driver) ReloadVM() error { + return d.sh("vim-cmd", "vmsvc/reload", d.vmId) +} + func (d *ESX5Driver) Start(vmxPathLocal string, headless bool) error { for i := 0; i < 20; i++ { err := d.sh("vim-cmd", "vmsvc/power.on", d.vmId) diff --git a/builder/vmware/iso/remote_driver.go b/builder/vmware/iso/remote_driver.go index b597d530a..7c62cd4d7 100644 --- a/builder/vmware/iso/remote_driver.go +++ b/builder/vmware/iso/remote_driver.go @@ -17,4 +17,10 @@ type RemoteDriver interface { // Removes a VM from inventory specified by the path to the VMX given. Unregister(string) error + + // Uploads a local file to remote side. + upload(dst, src string) error + + // Reload VM on remote side. + ReloadVM() error } diff --git a/builder/vmware/iso/remote_driver_mock.go b/builder/vmware/iso/remote_driver_mock.go index 233056ae3..2f4b3ae81 100644 --- a/builder/vmware/iso/remote_driver_mock.go +++ b/builder/vmware/iso/remote_driver_mock.go @@ -19,6 +19,10 @@ type RemoteDriverMock struct { UnregisterCalled bool UnregisterPath string UnregisterErr error + + uploadErr error + + ReloadVMErr error } func (d *RemoteDriverMock) UploadISO(path string, checksum string, checksumType string) (string, error) { @@ -38,3 +42,11 @@ func (d *RemoteDriverMock) Unregister(path string) error { d.UnregisterPath = path return d.UnregisterErr } + +func (d *RemoteDriverMock) upload(dst, src string) error { + return d.uploadErr +} + +func (d *RemoteDriverMock) ReloadVM() error { + return d.ReloadVMErr +} diff --git a/builder/vmware/iso/step_upload_vmx.go b/builder/vmware/iso/step_upload_vmx.go new file mode 100644 index 000000000..39e14af16 --- /dev/null +++ b/builder/vmware/iso/step_upload_vmx.go @@ -0,0 +1,48 @@ +package iso + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + vmwcommon "github.com/mitchellh/packer/builder/vmware/common" + "path/filepath" +) + + +// This step upload the VMX to the remote host +// +// Uses: +// driver Driver +// ui packer.Ui +// vmx_path string +// +// Produces: +// +type StepUploadVMX struct{ + RemoteType string +} + +func (c *StepUploadVMX) Run(state multistep.StateBag) multistep.StepAction { + driver := state.Get("driver").(vmwcommon.Driver) + + ui := state.Get("ui").(packer.Ui) + vmxPath := state.Get("vmx_path").(string) + + if c.RemoteType == "esx5" { + remoteDriver, ok := driver.(RemoteDriver) + if ok { + remoteVmxPath := filepath.ToSlash(filepath.Join(fmt.Sprintf("%s",remoteDriver), filepath.Base(vmxPath))) + if err := remoteDriver.upload(remoteVmxPath, vmxPath); err != nil { + state.Put("error", fmt.Errorf("Error writing VMX: %s", err)) + return multistep.ActionHalt + } + } + if err := remoteDriver.ReloadVM(); err != nil { + ui.Error(fmt.Sprintf("Error reload VM: %s", err)) + } + } + + return multistep.ActionContinue +} + +func (StepUploadVMX) Cleanup(multistep.StateBag) {} From 3d6865fafc394e888dea07abd7f2afcc524ecee7 Mon Sep 17 00:00:00 2001 From: Cheah Chu Yeow Date: Mon, 24 Nov 2014 08:14:05 +0000 Subject: [PATCH 02/86] builder/digitalocean: use DigitalOcean image ID for artifact Id(). --- builder/digitalocean/artifact.go | 4 ++-- builder/digitalocean/artifact_test.go | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/builder/digitalocean/artifact.go b/builder/digitalocean/artifact.go index 3f756a05d..d1d878193 100644 --- a/builder/digitalocean/artifact.go +++ b/builder/digitalocean/artifact.go @@ -3,6 +3,7 @@ package digitalocean import ( "fmt" "log" + "strconv" ) type Artifact struct { @@ -29,8 +30,7 @@ func (*Artifact) Files() []string { } func (a *Artifact) Id() string { - // mimicing the aws builder - return fmt.Sprintf("%s:%s", a.regionName, a.snapshotName) + return strconv.FormatUint(uint64(a.snapshotId), 10) } func (a *Artifact) String() string { diff --git a/builder/digitalocean/artifact_test.go b/builder/digitalocean/artifact_test.go index 83681b3fa..4492c7bf3 100644 --- a/builder/digitalocean/artifact_test.go +++ b/builder/digitalocean/artifact_test.go @@ -1,8 +1,9 @@ package digitalocean import ( - "github.com/mitchellh/packer/packer" "testing" + + "github.com/mitchellh/packer/packer" ) func TestArtifact_Impl(t *testing.T) { @@ -13,6 +14,15 @@ func TestArtifact_Impl(t *testing.T) { } } +func TestArtifactId(t *testing.T) { + a := &Artifact{"packer-foobar", 42, "San Francisco", nil} + expected := "42" + + if a.Id() != expected { + t.Fatalf("artifact ID should match: %v", expected) + } +} + func TestArtifactString(t *testing.T) { a := &Artifact{"packer-foobar", 42, "San Francisco", nil} expected := "A snapshot was created: 'packer-foobar' in region 'San Francisco'" From 1de1350ee2a4da47a27af12d19acb43b112b1a60 Mon Sep 17 00:00:00 2001 From: Emil Hessman Date: Mon, 24 Nov 2014 23:45:49 +0100 Subject: [PATCH 03/86] post-processor/vagrant: test for user variables in vagrantfile_template [GH-1125] Previously fixed in a47610879fcd5e2871b3e088e40d368a2915455f Closes #1125 --- post-processor/vagrant/post-processor_test.go | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/post-processor/vagrant/post-processor_test.go b/post-processor/vagrant/post-processor_test.go index 0ef151ebe..b035f39e0 100644 --- a/post-processor/vagrant/post-processor_test.go +++ b/post-processor/vagrant/post-processor_test.go @@ -4,6 +4,8 @@ import ( "bytes" "compress/flate" "github.com/mitchellh/packer/packer" + "io/ioutil" + "os" "strings" "testing" ) @@ -124,6 +126,41 @@ func TestPostProcessorPostProcess_badId(t *testing.T) { } } +func TestPostProcessorPostProcess_vagrantfileUserVariable(t *testing.T) { + var p PostProcessor + + f, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("err: %s", err) + } + defer os.Remove(f.Name()) + + c := map[string]interface{}{ + "packer_user_variables": map[string]string{ + "foo": f.Name(), + }, + + "vagrantfile_template": "{{user `foo`}}", + } + err = p.Configure(c) + if err != nil { + t.Fatalf("err: %s", err) + } + + a := &packer.MockArtifact{ + BuilderIdValue: "packer.parallels", + } + a2, _, err := p.PostProcess(testUi(), a) + if a2 != nil { + for _, fn := range a2.Files() { + defer os.Remove(fn) + } + } + if err != nil { + t.Fatalf("err: %s", err) + } +} + func TestProviderForName(t *testing.T) { if v, ok := providerForName("virtualbox").(*VBoxProvider); !ok { t.Fatalf("bad: %#v", v) From 3a8e02110e0d7c15cea572a3da5acdeedb9faeec Mon Sep 17 00:00:00 2001 From: Emil Hessman Date: Tue, 25 Nov 2014 07:04:17 +0100 Subject: [PATCH 04/86] post-processor/vagrant: process templates in Configure --- post-processor/vagrant/post-processor.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/post-processor/vagrant/post-processor.go b/post-processor/vagrant/post-processor.go index e43690e0c..9c8b9c75b 100644 --- a/post-processor/vagrant/post-processor.go +++ b/post-processor/vagrant/post-processor.go @@ -113,14 +113,8 @@ func (p *PostProcessor) PostProcessProvider(name string, provider Provider, ui p // Write our Vagrantfile var customVagrantfile string if config.VagrantfileTemplate != "" { - vagrantfilePath, err := config.tpl.Process(config.VagrantfileTemplate, nil) - if err != nil { - return nil, false, err - } - - ui.Message(fmt.Sprintf( - "Using custom Vagrantfile: %s", vagrantfilePath)) - customBytes, err := ioutil.ReadFile(vagrantfilePath) + ui.Message(fmt.Sprintf("Using custom Vagrantfile: %s", config.VagrantfileTemplate)) + customBytes, err := ioutil.ReadFile(config.VagrantfileTemplate) if err != nil { return nil, false, err } @@ -200,6 +194,17 @@ func (p *PostProcessor) configureSingle(config *Config, raws ...interface{}) err // Accumulate any errors errs := common.CheckUnusedConfig(md) + templates := map[string]*string{ + "vagrantfile_template": &config.VagrantfileTemplate, + } + + for key, ptr := range templates { + *ptr, err = config.tpl.Process(*ptr, nil) + if err != nil { + errs = packer.MultiErrorAppend(errs, fmt.Errorf("Error processing %s: %s", key, err)) + } + } + validates := map[string]*string{ "output": &config.OutputPath, "vagrantfile_template": &config.VagrantfileTemplate, From a65dc929f28ae6a53df8c04afc4dc8c84603dd09 Mon Sep 17 00:00:00 2001 From: Emil Hessman Date: Tue, 25 Nov 2014 12:41:24 +0100 Subject: [PATCH 05/86] post-processor/vagrant: test if vagrantfile_template exists Test for #801 --- post-processor/vagrant/post-processor_test.go | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/post-processor/vagrant/post-processor_test.go b/post-processor/vagrant/post-processor_test.go index b035f39e0..9ca31bf9c 100644 --- a/post-processor/vagrant/post-processor_test.go +++ b/post-processor/vagrant/post-processor_test.go @@ -84,16 +84,22 @@ func TestPostProcessorPrepare_outputPath(t *testing.T) { func TestPostProcessorPrepare_subConfigs(t *testing.T) { var p PostProcessor + f, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("err: %s", err) + } + defer os.Remove(f.Name()) + // Default c := testConfig() c["compression_level"] = 42 - c["vagrantfile_template"] = "foo" + c["vagrantfile_template"] = f.Name() c["override"] = map[string]interface{}{ "aws": map[string]interface{}{ "compression_level": 7, }, } - err := p.Configure(c) + err = p.Configure(c) if err != nil { t.Fatalf("err: %s", err) } @@ -102,7 +108,7 @@ func TestPostProcessorPrepare_subConfigs(t *testing.T) { t.Fatalf("bad: %#v", p.configs[""].CompressionLevel) } - if p.configs[""].VagrantfileTemplate != "foo" { + if p.configs[""].VagrantfileTemplate != f.Name() { t.Fatalf("bad: %#v", p.configs[""].VagrantfileTemplate) } @@ -110,11 +116,30 @@ func TestPostProcessorPrepare_subConfigs(t *testing.T) { t.Fatalf("bad: %#v", p.configs["aws"].CompressionLevel) } - if p.configs["aws"].VagrantfileTemplate != "foo" { + if p.configs["aws"].VagrantfileTemplate != f.Name() { t.Fatalf("bad: %#v", p.configs["aws"].VagrantfileTemplate) } } +func TestPostProcessorPrepare_vagrantfileTemplateExists(t *testing.T) { + var p PostProcessor + + f, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("err: %s", err) + } + + c := testConfig() + c["vagrantfile_template"] = f.Name() + + os.Remove(f.Name()) + + err = p.Configure(c) + if err == nil { + t.Fatal("expected an error since vagrantfile_template does not exist") + } +} + func TestPostProcessorPostProcess_badId(t *testing.T) { artifact := &packer.MockArtifact{ BuilderIdValue: "invalid.packer", From 53b1db16692c7bf2e654ac125b8676c02154d07a Mon Sep 17 00:00:00 2001 From: Emil Hessman Date: Tue, 25 Nov 2014 13:02:32 +0100 Subject: [PATCH 06/86] post-processor/vagrant: check if vagrantfile_template exists Fixes #801 --- post-processor/vagrant/post-processor.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/post-processor/vagrant/post-processor.go b/post-processor/vagrant/post-processor.go index 9c8b9c75b..50b80f7cf 100644 --- a/post-processor/vagrant/post-processor.go +++ b/post-processor/vagrant/post-processor.go @@ -210,6 +210,13 @@ func (p *PostProcessor) configureSingle(config *Config, raws ...interface{}) err "vagrantfile_template": &config.VagrantfileTemplate, } + if config.VagrantfileTemplate != "" { + _, err := os.Stat(config.VagrantfileTemplate) + if err != nil { + errs = packer.MultiErrorAppend(errs, fmt.Errorf("vagrantfile_template '%s' does not exist", config.VagrantfileTemplate)) + } + } + for n, ptr := range validates { if err := config.tpl.Validate(*ptr); err != nil { errs = packer.MultiErrorAppend( From 05affceaefbb8ce56fd0c3e5f61601252681d8ed Mon Sep 17 00:00:00 2001 From: Mikhail Zholobov Date: Thu, 27 Nov 2014 15:20:19 +0300 Subject: [PATCH 07/86] builder/parallels: Attach bootable ISO exactly to cdrom0 [GH-1667] --- builder/parallels/iso/step_attach_iso.go | 59 ++++++++---------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/builder/parallels/iso/step_attach_iso.go b/builder/parallels/iso/step_attach_iso.go index 428b12fdb..188846331 100644 --- a/builder/parallels/iso/step_attach_iso.go +++ b/builder/parallels/iso/step_attach_iso.go @@ -17,9 +17,8 @@ import ( // vmName string // // Produces: -type stepAttachISO struct { - cdromDevice string -} +// attachedIso bool +type stepAttachISO struct{} func (s *stepAttachISO) Run(state multistep.StateBag) multistep.StepAction { driver := state.Get("driver").(parallelscommon.Driver) @@ -27,22 +26,11 @@ func (s *stepAttachISO) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) vmName := state.Get("vmName").(string) - // Attach the disk to the controller - ui.Say("Attaching ISO to the new CD/DVD drive...") - cdrom, err := driver.DeviceAddCdRom(vmName, isoPath) - - if err != nil { - err := fmt.Errorf("Error attaching ISO: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - // Set new boot order ui.Say("Setting the boot order...") command := []string{ "set", vmName, - "--device-bootorder", fmt.Sprintf("hdd0 %s cdrom0 net0", cdrom), + "--device-bootorder", fmt.Sprintf("hdd0 cdrom0 net0"), } if err := driver.Prlctl(command...); err != nil { @@ -52,51 +40,42 @@ func (s *stepAttachISO) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - // Disable 'cdrom0' device - ui.Say("Disabling default CD/DVD drive...") + // Attach the disk to the cdrom0 device. We couldn't use a separated device because it is failed to boot in PD9 [GH-1667] + ui.Say("Attaching ISO to the default CD/DVD ROM device...") command = []string{ "set", vmName, - "--device-set", "cdrom0", "--disable", + "--device-set", "cdrom0", + "--image", isoPath, + "--enable", "--connect", } - if err := driver.Prlctl(command...); err != nil { - err := fmt.Errorf("Error disabling default CD/DVD drive: %s", err) + err := fmt.Errorf("Error attaching ISO: %s", err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - // Track the device name so that we can can delete later - s.cdromDevice = cdrom + // Set some state so we know to remove + state.Put("attachedIso", true) return multistep.ActionContinue } func (s *stepAttachISO) Cleanup(state multistep.StateBag) { + if _, ok := state.GetOk("attachedIso"); !ok { + return + } + driver := state.Get("driver").(parallelscommon.Driver) ui := state.Get("ui").(packer.Ui) vmName := state.Get("vmName").(string) - // Enable 'cdrom0' device back - log.Println("Enabling default CD/DVD drive...") + // Detach ISO by setting an empty string image. + log.Println("Detaching ISO from the default CD/DVD ROM device...") command := []string{ "set", vmName, - "--device-set", "cdrom0", "--enable", "--disconnect", - } - - if err := driver.Prlctl(command...); err != nil { - ui.Error(fmt.Sprintf("Error enabling default CD/DVD drive: %s", err)) - } - - // Detach ISO - if s.cdromDevice == "" { - return - } - - log.Println("Detaching ISO...") - command = []string{ - "set", vmName, - "--device-del", s.cdromDevice, + "--device-set", "cdrom0", + "--image", "", "--disconnect", "--enable", } if err := driver.Prlctl(command...); err != nil { From a5b21c59ba3012cc10bbffc6940916ba5e0de4b2 Mon Sep 17 00:00:00 2001 From: Mikhail Zholobov Date: Thu, 27 Nov 2014 17:13:50 +0300 Subject: [PATCH 08/86] builder/parallels: Move setting the boot order to the separated step --- builder/parallels/iso/builder.go | 1 + builder/parallels/iso/step_attach_iso.go | 16 +------- builder/parallels/iso/step_set_boot_order.go | 42 ++++++++++++++++++++ 3 files changed, 44 insertions(+), 15 deletions(-) create mode 100644 builder/parallels/iso/step_set_boot_order.go diff --git a/builder/parallels/iso/builder.go b/builder/parallels/iso/builder.go index 5b4c628a1..e5e42e8e7 100644 --- a/builder/parallels/iso/builder.go +++ b/builder/parallels/iso/builder.go @@ -256,6 +256,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe new(stepHTTPServer), new(stepCreateVM), new(stepCreateDisk), + new(stepSetBootOrder), new(stepAttachISO), ¶llelscommon.StepAttachParallelsTools{ ParallelsToolsMode: b.config.ParallelsToolsMode, diff --git a/builder/parallels/iso/step_attach_iso.go b/builder/parallels/iso/step_attach_iso.go index 188846331..049f2edf3 100644 --- a/builder/parallels/iso/step_attach_iso.go +++ b/builder/parallels/iso/step_attach_iso.go @@ -26,23 +26,9 @@ func (s *stepAttachISO) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) vmName := state.Get("vmName").(string) - // Set new boot order - ui.Say("Setting the boot order...") - command := []string{ - "set", vmName, - "--device-bootorder", fmt.Sprintf("hdd0 cdrom0 net0"), - } - - if err := driver.Prlctl(command...); err != nil { - err := fmt.Errorf("Error setting the boot order: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - // Attach the disk to the cdrom0 device. We couldn't use a separated device because it is failed to boot in PD9 [GH-1667] ui.Say("Attaching ISO to the default CD/DVD ROM device...") - command = []string{ + command := []string{ "set", vmName, "--device-set", "cdrom0", "--image", isoPath, diff --git a/builder/parallels/iso/step_set_boot_order.go b/builder/parallels/iso/step_set_boot_order.go new file mode 100644 index 000000000..0b98dfc68 --- /dev/null +++ b/builder/parallels/iso/step_set_boot_order.go @@ -0,0 +1,42 @@ +package iso + +import ( + "fmt" + "github.com/mitchellh/multistep" + parallelscommon "github.com/mitchellh/packer/builder/parallels/common" + "github.com/mitchellh/packer/packer" +) + +// This step sets the device boot order for the virtual machine. +// +// Uses: +// driver Driver +// ui packer.Ui +// vmName string +// +// Produces: +type stepSetBootOrder struct{} + +func (s *stepSetBootOrder) Run(state multistep.StateBag) multistep.StepAction { + driver := state.Get("driver").(parallelscommon.Driver) + ui := state.Get("ui").(packer.Ui) + vmName := state.Get("vmName").(string) + + // Set new boot order + ui.Say("Setting the boot order...") + command := []string{ + "set", vmName, + "--device-bootorder", fmt.Sprintf("hdd0 cdrom0 net0"), + } + + if err := driver.Prlctl(command...); err != nil { + err := fmt.Errorf("Error setting the boot order: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *stepSetBootOrder) Cleanup(state multistep.StateBag) {} From dbaf865e59c2a87198c461b79f57f0d6c8165947 Mon Sep 17 00:00:00 2001 From: Greg Cockburn Date: Mon, 1 Dec 2014 10:44:52 +1100 Subject: [PATCH 09/86] change ordering of ssh connection IP for amazon. fixes #1559 --- builder/amazon/common/ssh.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/ssh.go b/builder/amazon/common/ssh.go index 34fd746cb..f31437d89 100644 --- a/builder/amazon/common/ssh.go +++ b/builder/amazon/common/ssh.go @@ -16,14 +16,14 @@ func SSHAddress(e *ec2.EC2, port int, private bool) func(multistep.StateBag) (st for j := 0; j < 2; j++ { var host string i := state.Get("instance").(*ec2.Instance) - if i.DNSName != "" { - host = i.DNSName - } else if i.VpcId != "" { + if i.VpcId != "" { if i.PublicIpAddress != "" && !private { host = i.PublicIpAddress } else { host = i.PrivateIpAddress } + } else if i.DNSName != "" { + host = i.DNSName } if host != "" { From 075abfeeaed5f9b884bd0ec18edda8f9d1bec213 Mon Sep 17 00:00:00 2001 From: Godefroid Chapelle Date: Tue, 2 Dec 2014 14:58:13 +0100 Subject: [PATCH 10/86] Escapes cp alias Fix for #1656 --- builder/docker/communicator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/docker/communicator.go b/builder/docker/communicator.go index 36ba291d3..0956c82f2 100644 --- a/builder/docker/communicator.go +++ b/builder/docker/communicator.go @@ -75,7 +75,7 @@ func (c *Communicator) Upload(dst string, src io.Reader, fi *os.FileInfo) error // Copy the file into place by copying the temporary file we put // into the shared folder into the proper location in the container cmd := &packer.RemoteCmd{ - Command: fmt.Sprintf("cp %s/%s %s", c.ContainerDir, + Command: fmt.Sprintf("\cp %s/%s %s", c.ContainerDir, filepath.Base(tempfile.Name()), dst), } From 955122f2e1d4048c6f35c145d283d531de76da31 Mon Sep 17 00:00:00 2001 From: Godefroid Chapelle Date: Tue, 2 Dec 2014 15:07:29 +0100 Subject: [PATCH 11/86] Be more explicit about escaping alias --- builder/docker/communicator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/docker/communicator.go b/builder/docker/communicator.go index 0956c82f2..4c5b5043e 100644 --- a/builder/docker/communicator.go +++ b/builder/docker/communicator.go @@ -75,7 +75,7 @@ func (c *Communicator) Upload(dst string, src io.Reader, fi *os.FileInfo) error // Copy the file into place by copying the temporary file we put // into the shared folder into the proper location in the container cmd := &packer.RemoteCmd{ - Command: fmt.Sprintf("\cp %s/%s %s", c.ContainerDir, + Command: fmt.Sprintf("command cp %s/%s %s", c.ContainerDir, filepath.Base(tempfile.Name()), dst), } From 02ebef90316f84689906b2752a7efb63851a9b56 Mon Sep 17 00:00:00 2001 From: James Sinclair Date: Fri, 5 Dec 2014 17:27:00 +1100 Subject: [PATCH 12/86] Use an index loop as range loops over copies, not references Fixes #1637 --- builder/amazon/common/block_device.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builder/amazon/common/block_device.go b/builder/amazon/common/block_device.go index dc87c07fb..9557cc579 100644 --- a/builder/amazon/common/block_device.go +++ b/builder/amazon/common/block_device.go @@ -60,12 +60,12 @@ func (b *BlockDevices) Prepare(t *packer.ConfigTemplate) []error { var errs []error for outer, bds := range lists { - for i, bd := range bds { + for i := 0; i < len(bds); i++ { templates := map[string]*string{ - "device_name": &bd.DeviceName, - "snapshot_id": &bd.SnapshotId, - "virtual_name": &bd.VirtualName, - "volume_type": &bd.VolumeType, + "device_name": &bds[i].DeviceName, + "snapshot_id": &bds[i].SnapshotId, + "virtual_name": &bds[i].VirtualName, + "volume_type": &bds[i].VolumeType, } errs := make([]error, 0) From e4c544b3e21333c9f0c2465e851e43d09d1e4e3d Mon Sep 17 00:00:00 2001 From: Timur Batyrshin Date: Fri, 5 Dec 2014 20:26:41 +0300 Subject: [PATCH 13/86] added ssl_verify_mode parameter to chef-client provisioner --- provisioner/chef-client/provisioner.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/provisioner/chef-client/provisioner.go b/provisioner/chef-client/provisioner.go index 5d42fba26..b3d91b3e4 100644 --- a/provisioner/chef-client/provisioner.go +++ b/provisioner/chef-client/provisioner.go @@ -22,6 +22,7 @@ type Config struct { common.PackerConfig `mapstructure:",squash"` ChefEnvironment string `mapstructure:"chef_environment"` + SslVerifyMode string `mapstructure:"ssl_verify_mode"` ConfigTemplate string `mapstructure:"config_template"` ExecuteCommand string `mapstructure:"execute_command"` InstallCommand string `mapstructure:"install_command"` @@ -50,6 +51,7 @@ type ConfigTemplate struct { ValidationKeyPath string ValidationClientName string ChefEnvironment string + SslVerifyMode string } type ExecuteTemplate struct { @@ -79,6 +81,7 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { templates := map[string]*string{ "chef_environment": &p.config.ChefEnvironment, + "ssl_verify_mode": &p.config.SslVerifyMode, "config_template": &p.config.ConfigTemplate, "node_name": &p.config.NodeName, "staging_dir": &p.config.StagingDir, @@ -214,7 +217,7 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { } configPath, err := p.createConfig( - ui, comm, nodeName, serverUrl, remoteValidationKeyPath, p.config.ValidationClientName, p.config.ChefEnvironment) + ui, comm, nodeName, serverUrl, remoteValidationKeyPath, p.config.ValidationClientName, p.config.ChefEnvironment, p.config.SslVerifyMode) if err != nil { return fmt.Errorf("Error creating Chef config file: %s", err) } @@ -268,7 +271,7 @@ func (p *Provisioner) uploadDirectory(ui packer.Ui, comm packer.Communicator, ds return comm.UploadDir(dst, src, nil) } -func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, nodeName string, serverUrl string, remoteKeyPath string, validationClientName string, chefEnvironment string) (string, error) { +func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, nodeName string, serverUrl string, remoteKeyPath string, validationClientName string, chefEnvironment string, sslVerifyMode string) (string, error) { ui.Message("Creating configuration file 'client.rb'") // Read the template @@ -294,6 +297,7 @@ func (p *Provisioner) createConfig(ui packer.Ui, comm packer.Communicator, nodeN ValidationKeyPath: remoteKeyPath, ValidationClientName: validationClientName, ChefEnvironment: chefEnvironment, + SslVerifyMode: sslVerifyMode, }) if err != nil { return "", err @@ -574,4 +578,7 @@ node_name "{{.NodeName}}" {{if ne .ChefEnvironment ""}} environment "{{.ChefEnvironment}}" {{end}} +{{if ne .SslVerifyMode ""}} +ssl_verify_mode :{{.SslVerifyMode}} +{{end}} ` From 44a2ceb3d89fc75a6b96dfb3cd4feb368a91c420 Mon Sep 17 00:00:00 2001 From: Matt Rogers Date: Fri, 28 Nov 2014 10:57:46 -0600 Subject: [PATCH 14/86] Use the latest micro instance from the free tier --- website/source/intro/getting-started/build-image.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/intro/getting-started/build-image.html.markdown b/website/source/intro/getting-started/build-image.html.markdown index 024447c78..a329059ad 100644 --- a/website/source/intro/getting-started/build-image.html.markdown +++ b/website/source/intro/getting-started/build-image.html.markdown @@ -55,7 +55,7 @@ briefly. Create a file `example.json` and fill it with the following contents: "secret_key": "{{user `aws_secret_key`}}", "region": "us-east-1", "source_ami": "ami-de0d9eb7", - "instance_type": "t1.micro", + "instance_type": "t2.micro", "ssh_username": "ubuntu", "ami_name": "packer-example {{timestamp}}" }] From df48ec33568289de3acc76e81c7031b663a133fe Mon Sep 17 00:00:00 2001 From: Matt Rogers Date: Fri, 28 Nov 2014 10:59:07 -0600 Subject: [PATCH 15/86] Use a newer source AMI The AMI that was previously listed before this change no longer exists. --- website/source/intro/getting-started/build-image.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/intro/getting-started/build-image.html.markdown b/website/source/intro/getting-started/build-image.html.markdown index a329059ad..ec6861d2e 100644 --- a/website/source/intro/getting-started/build-image.html.markdown +++ b/website/source/intro/getting-started/build-image.html.markdown @@ -54,7 +54,7 @@ briefly. Create a file `example.json` and fill it with the following contents: "access_key": "{{user `aws_access_key`}}", "secret_key": "{{user `aws_secret_key`}}", "region": "us-east-1", - "source_ami": "ami-de0d9eb7", + "source_ami": "ami-3d50120d", "instance_type": "t2.micro", "ssh_username": "ubuntu", "ami_name": "packer-example {{timestamp}}" From 7c67228912d6f7ae8bc74ef43dea24642aa5bc81 Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Tue, 9 Dec 2014 08:42:33 -0800 Subject: [PATCH 16/86] Check if image already exists before doing anything else on GCE. This fixes #1729. --- builder/googlecompute/builder.go | 1 + builder/googlecompute/driver.go | 4 +++ builder/googlecompute/driver_gce.go | 9 ++++- builder/googlecompute/driver_mock.go | 8 +++++ .../step_check_existing_image.go | 33 +++++++++++++++++++ .../step_check_existing_image_test.go | 32 ++++++++++++++++++ 6 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 builder/googlecompute/step_check_existing_image.go create mode 100644 builder/googlecompute/step_check_existing_image_test.go diff --git a/builder/googlecompute/builder.go b/builder/googlecompute/builder.go index 9c25490cb..cfb9c6e56 100644 --- a/builder/googlecompute/builder.go +++ b/builder/googlecompute/builder.go @@ -49,6 +49,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps. steps := []multistep.Step{ + new(StepCheckExistingImage), &StepCreateSSHKey{ Debug: b.config.PackerDebug, DebugKeyPath: fmt.Sprintf("gce_%s.pem", b.config.PackerBuildName), diff --git a/builder/googlecompute/driver.go b/builder/googlecompute/driver.go index 046bfa3e2..6f035c562 100644 --- a/builder/googlecompute/driver.go +++ b/builder/googlecompute/driver.go @@ -4,6 +4,10 @@ package googlecompute // with GCE. The Driver interface exists mostly to allow a mock implementation // to be used to test the steps. type Driver interface { + // ImageExists returns true if the specified image exists. If an error + // occurs calling the API, this method returns false. + ImageExists(name string) bool + // CreateImage creates an image from the given disk in Google Compute // Engine. CreateImage(name, description, zone, disk string) <-chan error diff --git a/builder/googlecompute/driver_gce.go b/builder/googlecompute/driver_gce.go index 835add5d9..674019f52 100644 --- a/builder/googlecompute/driver_gce.go +++ b/builder/googlecompute/driver_gce.go @@ -7,9 +7,9 @@ import ( "time" "code.google.com/p/google-api-go-client/compute/v1" + "github.com/mitchellh/packer/packer" "golang.org/x/oauth2" "golang.org/x/oauth2/google" - "github.com/mitchellh/packer/packer" ) // driverGCE is a Driver implementation that actually talks to GCE. @@ -60,6 +60,13 @@ func NewDriverGCE(ui packer.Ui, p string, a *accountFile) (Driver, error) { }, nil } +func (d *driverGCE) ImageExists(name string) bool { + _, err := d.service.Images.Get(d.projectId, name).Do() + // The API may return an error for reasons other than the image not + // existing, but this heuristic is sufficient for now. + return err == nil +} + func (d *driverGCE) CreateImage(name, description, zone, disk string) <-chan error { image := &compute.Image{ Description: description, diff --git a/builder/googlecompute/driver_mock.go b/builder/googlecompute/driver_mock.go index 70b310e39..1a9c03ae9 100644 --- a/builder/googlecompute/driver_mock.go +++ b/builder/googlecompute/driver_mock.go @@ -3,6 +3,9 @@ package googlecompute // DriverMock is a Driver implementation that is a mocked out so that // it can be used for tests. type DriverMock struct { + ImageExistsName string + ImageExistsResult bool + CreateImageName string CreateImageDesc string CreateImageZone string @@ -37,6 +40,11 @@ type DriverMock struct { WaitForInstanceErrCh <-chan error } +func (d *DriverMock) ImageExists(name string) bool { + d.ImageExistsName = name + return d.ImageExistsResult +} + func (d *DriverMock) CreateImage(name, description, zone, disk string) <-chan error { d.CreateImageName = name d.CreateImageDesc = description diff --git a/builder/googlecompute/step_check_existing_image.go b/builder/googlecompute/step_check_existing_image.go new file mode 100644 index 000000000..2a3365739 --- /dev/null +++ b/builder/googlecompute/step_check_existing_image.go @@ -0,0 +1,33 @@ +package googlecompute + +import ( + "fmt" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" +) + +// StepCheckExistingImage represents a Packer build step that checks if the +// target image already exists, and aborts immediately if so. +type StepCheckExistingImage int + +// Run executes the Packer build step that checks if the image already exists. +func (s *StepCheckExistingImage) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(*Config) + driver := state.Get("driver").(Driver) + ui := state.Get("ui").(packer.Ui) + + ui.Say("Checking image does not exist...") + exists := driver.ImageExists(config.ImageName) + if exists { + err := fmt.Errorf("Image %s already exists", config.ImageName) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +// Cleanup. +func (s *StepCheckExistingImage) Cleanup(state multistep.StateBag) {} diff --git a/builder/googlecompute/step_check_existing_image_test.go b/builder/googlecompute/step_check_existing_image_test.go new file mode 100644 index 000000000..a7b336407 --- /dev/null +++ b/builder/googlecompute/step_check_existing_image_test.go @@ -0,0 +1,32 @@ +package googlecompute + +import ( + "github.com/mitchellh/multistep" + "testing" +) + +func TestStepCheckExistingImage_impl(t *testing.T) { + var _ multistep.Step = new(StepCheckExistingImage) +} + +func TestStepCheckExistingImage(t *testing.T) { + state := testState(t) + step := new(StepCheckExistingImage) + defer step.Cleanup(state) + + state.Put("instance_name", "foo") + + config := state.Get("config").(*Config) + driver := state.Get("driver").(*DriverMock) + driver.ImageExistsResult = true + + // run the step + if action := step.Run(state); action != multistep.ActionHalt { + t.Fatalf("bad action: %#v", action) + } + + // Verify state + if driver.ImageExistsName != config.ImageName { + t.Fatalf("bad: %#v", driver.ImageExistsName) + } +} From fe47f45800c6854ec145d9453bdf6c3dfb8957aa Mon Sep 17 00:00:00 2001 From: Dustin Carlino Date: Tue, 9 Dec 2014 08:44:01 -0800 Subject: [PATCH 17/86] Remove bucket_name from GCE config. Fixes #1732. Forgot to do this in 23c947acf049b734c03ff3e24530f0310475b1a5. --- builder/googlecompute/config.go | 7 ------- builder/googlecompute/config_test.go | 12 ------------ website/source/docs/builders/googlecompute.markdown | 4 ---- 3 files changed, 23 deletions(-) diff --git a/builder/googlecompute/config.go b/builder/googlecompute/config.go index 27619c9bf..96557da10 100644 --- a/builder/googlecompute/config.go +++ b/builder/googlecompute/config.go @@ -19,7 +19,6 @@ type Config struct { AccountFile string `mapstructure:"account_file"` ProjectId string `mapstructure:"project_id"` - BucketName string `mapstructure:"bucket_name"` DiskName string `mapstructure:"disk_name"` DiskSizeGb int64 `mapstructure:"disk_size"` ImageName string `mapstructure:"image_name"` @@ -109,7 +108,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { templates := map[string]*string{ "account_file": &c.AccountFile, - "bucket_name": &c.BucketName, "disk_name": &c.DiskName, "image_name": &c.ImageName, "image_description": &c.ImageDescription, @@ -135,11 +133,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { } // Process required parameters. - if c.BucketName == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("a bucket_name must be specified")) - } - if c.ProjectId == "" { errs = packer.MultiErrorAppend( errs, errors.New("a project_id must be specified")) diff --git a/builder/googlecompute/config_test.go b/builder/googlecompute/config_test.go index 77dcca4c2..4a7a1ed67 100644 --- a/builder/googlecompute/config_test.go +++ b/builder/googlecompute/config_test.go @@ -8,7 +8,6 @@ import ( func testConfig(t *testing.T) map[string]interface{} { return map[string]interface{}{ "account_file": testAccountFile(t), - "bucket_name": "foo", "project_id": "hashicorp", "source_image": "foo", "zone": "us-east-1a", @@ -57,17 +56,6 @@ func TestConfigPrepare(t *testing.T) { true, }, - { - "bucket_name", - nil, - true, - }, - { - "bucket_name", - "good", - false, - }, - { "private_key_file", "/tmp/i/should/not/exist", diff --git a/website/source/docs/builders/googlecompute.markdown b/website/source/docs/builders/googlecompute.markdown index 56d568ac0..0481fcf8e 100644 --- a/website/source/docs/builders/googlecompute.markdown +++ b/website/source/docs/builders/googlecompute.markdown @@ -73,7 +73,6 @@ existing GCE image. The account file is obtained in the previous section. ```javascript { "type": "googlecompute", - "bucket_name": "my-project-packer-images", "account_file": "account.json", "project_id": "my-project", "source_image": "debian-7-wheezy-v20140718", @@ -88,9 +87,6 @@ each category, the available options are alphabetized and described. ### Required: -* `bucket_name` (string) - The Google Cloud Storage bucket to store the - images that are created. The bucket must already exist in your project - * `project_id` (string) - The project ID that will be used to launch instances and store images. From 76a09e0295c53f2e02c1bfa492e824b38eff7464 Mon Sep 17 00:00:00 2001 From: Armon Dadgar Date: Tue, 9 Dec 2014 15:29:36 -0800 Subject: [PATCH 18/86] CHANGELOG updates --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f24003bea..f4f9c78f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,33 @@ ## 0.8.0 (unreleased) +FEATURES: +IMPROVEMENTS: + +* builder/googlecompute: Support for ubuntu-os-cloud project +* builder/googlecompute: Support for OAuth2 to avoid client secrets file +* builder/googlecompute: GCE image from persistant disk instead of tarball +* builder/qemu: Checksum type "none" can be used +* provisioner/chef: Generate a node name if none available +* provisioner/chef: Added ssl_verify_mode configuration + +BUG FIXES: + +* builder/parallels: Fixed attachment of ISO to cdrom device +* builder/parallels: Fixed boot load ordering +* builder/digitalocean: Fixed decoding of size +* builder/digitalocean: Fixed missing content-type header in request +* builder/digitalocean: Fixed use of private IP +* builder/digitalocean: Fixed the artifact ID generation +* builder/vsphere: Fixed credential escaping +* builder/qemu: Fixed use of CDROM with disk_image +* builder/aws: Fixed IP address for SSH in VPC +* builder/aws: Fixed issue with multiple block devices +* builder/vmware: Upload VMX to ESX5 after editing +* communicator/docker: Fix handling of symlinks during upload +* provisioner/chef: Fixed use of sudo in some cases +* core: Fixed build name interpolation +* postprocessor/vagrant: Fixed check for Vagrantfile template ## 0.7.2 (October 28, 2014) From 835cadeee2dc959576b450a72f9103fbb695eaec Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 28 Oct 2014 14:55:32 -0700 Subject: [PATCH 19/86] packer: push configuration --- packer/template.go | 11 +++++++++++ packer/template_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/packer/template.go b/packer/template.go index 205a54c32..50b1fffd6 100644 --- a/packer/template.go +++ b/packer/template.go @@ -24,6 +24,7 @@ type rawTemplate struct { Description string Builders []map[string]interface{} Hooks map[string][]string + Push PushConfig PostProcessors []interface{} `mapstructure:"post-processors"` Provisioners []map[string]interface{} Variables map[string]interface{} @@ -36,10 +37,19 @@ type Template struct { Variables map[string]RawVariable Builders map[string]RawBuilderConfig Hooks map[string][]string + Push *PushConfig PostProcessors [][]RawPostProcessorConfig Provisioners []RawProvisionerConfig } +// PushConfig is the configuration structure for the push settings. +type PushConfig struct { + Name string + Include []string + Exclude []string + VCS bool +} + // The RawBuilderConfig struct represents a raw, unprocessed builder // configuration. It contains the name of the builder as well as the // raw configuration. If requested, this is used to compile into a full @@ -154,6 +164,7 @@ func ParseTemplate(data []byte, vars map[string]string) (t *Template, err error) t.Variables = make(map[string]RawVariable) t.Builders = make(map[string]RawBuilderConfig) t.Hooks = rawTpl.Hooks + t.Push = &rawTpl.Push t.PostProcessors = make([][]RawPostProcessorConfig, len(rawTpl.PostProcessors)) t.Provisioners = make([]RawProvisionerConfig, len(rawTpl.Provisioners)) diff --git a/packer/template_test.go b/packer/template_test.go index 6feff138a..d42895665 100644 --- a/packer/template_test.go +++ b/packer/template_test.go @@ -541,6 +541,41 @@ func TestParseTemplate_ProvisionerPauseBefore(t *testing.T) { } } +func TestParseTemplateFile_push(t *testing.T) { + data := ` + { + "builders": [{"type": "something"}], + + "push": { + "name": "hello", + "include": ["one"], + "exclude": ["two"] + } + } + ` + + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("err: %s", err) + } + tf.Write([]byte(data)) + tf.Close() + + result, err := ParseTemplateFile(tf.Name(), nil) + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := &PushConfig{ + Name: "hello", + Include: []string{"one"}, + Exclude: []string{"two"}, + } + if !reflect.DeepEqual(result.Push, expected) { + t.Fatalf("bad: %#v", result.Push) + } +} + func TestParseTemplate_Variables(t *testing.T) { data := ` { From 6fb375c89ee48439a6df1f7d4a603dd1778e4b02 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 28 Oct 2014 15:09:22 -0700 Subject: [PATCH 20/86] command: push command --- command/command_test.go | 13 ++++++++++ command/push.go | 55 +++++++++++++++++++++++++++++++++++++++++ command/push_test.go | 21 ++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 command/command_test.go create mode 100644 command/push.go create mode 100644 command/push_test.go diff --git a/command/command_test.go b/command/command_test.go new file mode 100644 index 000000000..7574d663d --- /dev/null +++ b/command/command_test.go @@ -0,0 +1,13 @@ +package command + +import ( + "testing" + + "github.com/mitchellh/cli" +) + +func testMeta(t *testing.T) Meta { + return Meta{ + Ui: new(cli.MockUi), + } +} diff --git a/command/push.go b/command/push.go new file mode 100644 index 000000000..d50d549d1 --- /dev/null +++ b/command/push.go @@ -0,0 +1,55 @@ +package command + +import ( + "flag" + "fmt" + "strings" + + "github.com/mitchellh/packer/packer" +) + +type PushCommand struct { + Meta +} + +func (c *PushCommand) Run(args []string) int { + f := flag.NewFlagSet("push", flag.ContinueOnError) + f.Usage = func() { c.Ui.Error(c.Help()) } + if err := f.Parse(args); err != nil { + return 1 + } + + args = f.Args() + if len(args) != 1 { + f.Usage() + return 1 + } + + // Read the template + tpl, err := packer.ParseTemplateFile(args[0], nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Failed to parse template: %s", err)) + return 1 + } + + // TODO: validate the template + println(tpl.Push.Name) + + return 0 +} + +func (*PushCommand) Help() string { + helpText := ` +Usage: packer push [options] TEMPLATE + + Push the template and the files it needs to a Packer build service. + This will not initiate any builds, it will only update the templates + used for builds. +` + + return strings.TrimSpace(helpText) +} + +func (*PushCommand) Synopsis() string { + return "push template files to a Packer build service" +} diff --git a/command/push_test.go b/command/push_test.go new file mode 100644 index 000000000..b7e1f800a --- /dev/null +++ b/command/push_test.go @@ -0,0 +1,21 @@ +package command + +import ( + "testing" +) + +func TestPush_noArgs(t *testing.T) { + c := &PushCommand{Meta: testMeta(t)} + code := c.Run(nil) + if code != 1 { + t.Fatalf("bad: %#v", code) + } +} + +func TestPush_multiArgs(t *testing.T) { + c := &PushCommand{Meta: testMeta(t)} + code := c.Run([]string{"one", "two"}) + if code != 1 { + t.Fatalf("bad: %#v", code) + } +} From 84c8344794bb83e493f039008a18e12f8829f866 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 28 Oct 2014 15:57:22 -0700 Subject: [PATCH 21/86] hook up push to commands --- commands.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/commands.go b/commands.go index 193fd1465..9c6458f64 100644 --- a/commands.go +++ b/commands.go @@ -50,6 +50,12 @@ func init() { }, nil }, + "push": func() (cli.Command, error) { + return &command.PushCommand{ + Meta: meta, + }, nil + }, + "validate": func() (cli.Command, error) { return &command.ValidateCommand{ Meta: meta, From fbc1551048f538a0854090a5004f9e9362c74719 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 28 Oct 2014 19:29:51 -0700 Subject: [PATCH 22/86] command/push: partially implemented, tests --- command/command_test.go | 15 +++ command/push.go | 94 +++++++++++++- command/push_test.go | 122 ++++++++++++++++++ .../test-fixtures/push-no-name/template.json | 3 + command/test-fixtures/push/template.json | 7 + packer/template.go | 1 + 6 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 command/test-fixtures/push-no-name/template.json create mode 100644 command/test-fixtures/push/template.json diff --git a/command/command_test.go b/command/command_test.go index 7574d663d..500ea7f9e 100644 --- a/command/command_test.go +++ b/command/command_test.go @@ -1,11 +1,26 @@ package command import ( + "path/filepath" "testing" "github.com/mitchellh/cli" ) +const fixturesDir = "./test-fixtures" + +func fatalCommand(t *testing.T, m Meta) { + ui := m.Ui.(*cli.MockUi) + t.Fatalf( + "Bad exit code.\n\nStdout:\n\n%s\n\nStderr:\n\n%s", + ui.OutputWriter.String(), + ui.ErrorWriter.String()) +} + +func testFixture(n string) string { + return filepath.Join(fixturesDir, n) +} + func testMeta(t *testing.T) Meta { return Meta{ Ui: new(cli.MockUi), diff --git a/command/push.go b/command/push.go index d50d549d1..1b43be6b9 100644 --- a/command/push.go +++ b/command/push.go @@ -3,18 +3,30 @@ package command import ( "flag" "fmt" + "io" + "path/filepath" "strings" + "github.com/hashicorp/harmony-go/archive" "github.com/mitchellh/packer/packer" ) +// archiveTemplateEntry is the name the template always takes within the slug. +const archiveTemplateEntry = ".packer-template.json" + type PushCommand struct { Meta + + // For tests: + uploadFn func(io.Reader, *uploadOpts) (<-chan struct{}, <-chan error, error) } func (c *PushCommand) Run(args []string) int { + var token string + f := flag.NewFlagSet("push", flag.ContinueOnError) f.Usage = func() { c.Ui.Error(c.Help()) } + f.StringVar(&token, "token", "", "token") if err := f.Parse(args); err != nil { return 1 } @@ -32,8 +44,66 @@ func (c *PushCommand) Run(args []string) int { return 1 } - // TODO: validate the template - println(tpl.Push.Name) + // Validate some things + if tpl.Push.Name == "" { + c.Ui.Error(fmt.Sprintf( + "The 'push' section must be specified in the template with\n" + + "at least the 'name' option set.")) + return 1 + } + + // Build the archiving options + var opts archive.ArchiveOpts + opts.Include = tpl.Push.Include + opts.Exclude = tpl.Push.Exclude + opts.VCS = tpl.Push.VCS + opts.Extra = map[string]string{ + archiveTemplateEntry: args[0], + } + + // Determine the path we're archiving + path := tpl.Push.BaseDir + if path == "" { + path, err = filepath.Abs(args[0]) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error determining path to archive: %s", err)) + return 1 + } + path = filepath.Dir(path) + } + + // Build the upload options + var uploadOpts uploadOpts + uploadOpts.Slug = tpl.Push.Name + uploadOpts.Token = token + + // Start the archiving process + r, archiveErrCh, err := archive.Archive(path, &opts) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error archiving: %s", err)) + return 1 + } + + // Start the upload process + doneCh, uploadErrCh, err := c.upload(r, &uploadOpts) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error starting upload: %s", err)) + return 1 + } + + err = nil + select { + case err = <-archiveErrCh: + err = fmt.Errorf("Error archiving: %s", err) + case err = <-uploadErrCh: + err = fmt.Errorf("Error uploading: %s", err) + case <-doneCh: + } + + if err != nil { + c.Ui.Error(err.Error()) + return 1 + } return 0 } @@ -45,6 +115,11 @@ Usage: packer push [options] TEMPLATE Push the template and the files it needs to a Packer build service. This will not initiate any builds, it will only update the templates used for builds. + +Options: + + -token= Access token to use to upload. If blank, the + TODO environmental variable will be used. ` return strings.TrimSpace(helpText) @@ -53,3 +128,18 @@ Usage: packer push [options] TEMPLATE func (*PushCommand) Synopsis() string { return "push template files to a Packer build service" } + +func (c *PushCommand) upload( + r io.Reader, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { + if c.uploadFn != nil { + return c.uploadFn(r, opts) + } + + return nil, nil, nil +} + +type uploadOpts struct { + URL string + Slug string + Token string +} diff --git a/command/push_test.go b/command/push_test.go index b7e1f800a..b57276e5f 100644 --- a/command/push_test.go +++ b/command/push_test.go @@ -1,6 +1,14 @@ package command import ( + "fmt" + "archive/tar" + "bytes" + "compress/gzip" + "io" + "path/filepath" + "reflect" + "sort" "testing" ) @@ -19,3 +27,117 @@ func TestPush_multiArgs(t *testing.T) { t.Fatalf("bad: %#v", code) } } + +func TestPush(t *testing.T) { + var actualR io.Reader + var actualOpts *uploadOpts + uploadFn := func(r io.Reader, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { + actualR = r + actualOpts = opts + + doneCh := make(chan struct{}) + close(doneCh) + return doneCh, nil, nil + } + + c := &PushCommand{ + Meta: testMeta(t), + uploadFn: uploadFn, + } + + args := []string{filepath.Join(testFixture("push"), "template.json")} + if code := c.Run(args); code != 0 { + fatalCommand(t, c.Meta) + } + + actual := testArchive(t, actualR) + expected := []string{ + archiveTemplateEntry, + "template.json", + } + + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("bad: %#v", actual) + } +} + +func TestPush_noName(t *testing.T) { + uploadFn := func(r io.Reader, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { + return nil, nil, nil + } + + c := &PushCommand{ + Meta: testMeta(t), + uploadFn: uploadFn, + } + + args := []string{filepath.Join(testFixture("push-no-name"), "template.json")} + if code := c.Run(args); code != 1 { + fatalCommand(t, c.Meta) + } +} + +func TestPush_uploadError(t *testing.T) { + uploadFn := func(r io.Reader, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { + return nil, nil, fmt.Errorf("bad") + } + + c := &PushCommand{ + Meta: testMeta(t), + uploadFn: uploadFn, + } + + args := []string{filepath.Join(testFixture("push"), "template.json")} + if code := c.Run(args); code != 1 { + fatalCommand(t, c.Meta) + } +} + +func TestPush_uploadErrorCh(t *testing.T) { + uploadFn := func(r io.Reader, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { + errCh := make(chan error, 1) + errCh <- fmt.Errorf("bad") + return nil, errCh, nil + } + + c := &PushCommand{ + Meta: testMeta(t), + uploadFn: uploadFn, + } + + args := []string{filepath.Join(testFixture("push"), "template.json")} + if code := c.Run(args); code != 1 { + fatalCommand(t, c.Meta) + } +} + +func testArchive(t *testing.T, r io.Reader) []string { + // Finish the archiving process in-memory + var buf bytes.Buffer + if _, err := io.Copy(&buf, r); err != nil { + t.Fatalf("err: %s", err) + } + + gzipR, err := gzip.NewReader(&buf) + if err != nil { + t.Fatalf("err: %s", err) + } + tarR := tar.NewReader(gzipR) + + // Read all the entries + result := make([]string, 0, 5) + for { + hdr, err := tarR.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("err: %s", err) + } + + result = append(result, hdr.Name) + } + + sort.Strings(result) + return result +} diff --git a/command/test-fixtures/push-no-name/template.json b/command/test-fixtures/push-no-name/template.json new file mode 100644 index 000000000..5b5ede505 --- /dev/null +++ b/command/test-fixtures/push-no-name/template.json @@ -0,0 +1,3 @@ +{ + "builders": [{"type": "dummy"}] +} diff --git a/command/test-fixtures/push/template.json b/command/test-fixtures/push/template.json new file mode 100644 index 000000000..63b0f9037 --- /dev/null +++ b/command/test-fixtures/push/template.json @@ -0,0 +1,7 @@ +{ + "builders": [{"type": "dummy"}], + + "push": { + "name": "foo/bar" + } +} diff --git a/packer/template.go b/packer/template.go index 50b1fffd6..562d23edd 100644 --- a/packer/template.go +++ b/packer/template.go @@ -45,6 +45,7 @@ type Template struct { // PushConfig is the configuration structure for the push settings. type PushConfig struct { Name string + BaseDir string Include []string Exclude []string VCS bool From 0f5ef2ce2bdedd08d29ed8579f8b386d557cf4a8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 28 Oct 2014 19:34:19 -0700 Subject: [PATCH 23/86] command/push: close reader --- command/push.go | 1 + command/push_test.go | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/command/push.go b/command/push.go index 1b43be6b9..dc4cd783f 100644 --- a/command/push.go +++ b/command/push.go @@ -83,6 +83,7 @@ func (c *PushCommand) Run(args []string) int { c.Ui.Error(fmt.Sprintf("Error archiving: %s", err)) return 1 } + defer r.Close() // Start the upload process doneCh, uploadErrCh, err := c.upload(r, &uploadOpts) diff --git a/command/push_test.go b/command/push_test.go index b57276e5f..2d8222cfa 100644 --- a/command/push_test.go +++ b/command/push_test.go @@ -29,10 +29,10 @@ func TestPush_multiArgs(t *testing.T) { } func TestPush(t *testing.T) { - var actualR io.Reader + var actual []string var actualOpts *uploadOpts uploadFn := func(r io.Reader, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { - actualR = r + actual = testArchive(t, r) actualOpts = opts doneCh := make(chan struct{}) @@ -50,7 +50,6 @@ func TestPush(t *testing.T) { fatalCommand(t, c.Meta) } - actual := testArchive(t, actualR) expected := []string{ archiveTemplateEntry, "template.json", From 2f60707463ade25805c32df79aa20cc637949c4a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 28 Oct 2014 19:43:41 -0700 Subject: [PATCH 24/86] command/push: actual upload to named build config --- command/push.go | 57 ++++++++++++++++++++++++++++++++++++++++---- command/push_test.go | 5 ++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/command/push.go b/command/push.go index dc4cd783f..1068a2243 100644 --- a/command/push.go +++ b/command/push.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/hashicorp/harmony-go" "github.com/hashicorp/harmony-go/archive" "github.com/mitchellh/packer/packer" ) @@ -17,6 +18,8 @@ const archiveTemplateEntry = ".packer-template.json" type PushCommand struct { Meta + client *harmony.Client + // For tests: uploadFn func(io.Reader, *uploadOpts) (<-chan struct{}, <-chan error, error) } @@ -52,6 +55,10 @@ func (c *PushCommand) Run(args []string) int { return 1 } + // Build our client + c.client = harmony.DefaultClient() + defer func() { c.client = nil }() + // Build the archiving options var opts archive.ArchiveOpts opts.Include = tpl.Push.Include @@ -76,6 +83,10 @@ func (c *PushCommand) Run(args []string) int { var uploadOpts uploadOpts uploadOpts.Slug = tpl.Push.Name uploadOpts.Token = token + uploadOpts.Builds = make(map[string]string) + for _, b := range tpl.Builders { + uploadOpts.Builds[b.Name] = b.Type + } // Start the archiving process r, archiveErrCh, err := archive.Archive(path, &opts) @@ -136,11 +147,49 @@ func (c *PushCommand) upload( return c.uploadFn(r, opts) } - return nil, nil, nil + // Separate the slug into the user and name components + user, name, err := harmony.ParseSlug(opts.Slug) + if err != nil { + return nil, nil, fmt.Errorf("upload: %s", err) + } + + // Get the app + bc, err := c.client.BuildConfig(user, name) + if err != nil { + return nil, nil, fmt.Errorf("upload: %s", err) + } + + // Build the version to send up + version := harmony.BuildConfigVersion{ + User: bc.User, + Name: bc.Name, + Builds: make([]harmony.BuildConfigBuild, 0, len(opts.Builds)), + } + for name, t := range opts.Builds { + version.Builds = append(version.Builds, harmony.BuildConfigBuild{ + Name: name, + Type: t, + }) + } + + // Start the upload + doneCh, errCh := make(chan struct{}), make(chan error) + go func() { + err := c.client.UploadBuildConfigVersion(&version, r) + if err != nil { + errCh <- err + return + } + + close(doneCh) + }() + + return doneCh, errCh, nil } type uploadOpts struct { - URL string - Slug string - Token string + URL string + Slug string + Token string + Builds map[string]string } diff --git a/command/push_test.go b/command/push_test.go index 2d8222cfa..6ea1ec326 100644 --- a/command/push_test.go +++ b/command/push_test.go @@ -58,6 +58,11 @@ func TestPush(t *testing.T) { if !reflect.DeepEqual(actual, expected) { t.Fatalf("bad: %#v", actual) } + + expectedBuilds := map[string]string{"dummy": "dummy"} + if !reflect.DeepEqual(actualOpts.Builds, expectedBuilds) { + t.Fatalf("bad: %#v", actualOpts.Builds) + } } func TestPush_noName(t *testing.T) { From 2f65b1bb670f6385a39c24f0a71cf2d259a549a1 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 28 Oct 2014 20:02:07 -0700 Subject: [PATCH 25/86] command/push: create the build config if we can --- command/push.go | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/command/push.go b/command/push.go index 1068a2243..9e7c735bb 100644 --- a/command/push.go +++ b/command/push.go @@ -13,7 +13,7 @@ import ( ) // archiveTemplateEntry is the name the template always takes within the slug. -const archiveTemplateEntry = ".packer-template.json" +const archiveTemplateEntry = ".packer-template" type PushCommand struct { Meta @@ -25,10 +25,12 @@ type PushCommand struct { } func (c *PushCommand) Run(args []string) int { + var create bool var token string f := flag.NewFlagSet("push", flag.ContinueOnError) f.Usage = func() { c.Ui.Error(c.Help()) } + f.BoolVar(&create, "create", false, "create") f.StringVar(&token, "token", "", "token") if err := f.Parse(args); err != nil { return 1 @@ -88,6 +90,12 @@ func (c *PushCommand) Run(args []string) int { uploadOpts.Builds[b.Name] = b.Type } + // Create the build config if it doesn't currently exist. + if err := c.create(uploadOpts.Slug, create); err != nil { + c.Ui.Error(err.Error()) + return 1 + } + // Start the archiving process r, archiveErrCh, err := archive.Archive(path, &opts) if err != nil { @@ -128,8 +136,13 @@ Usage: packer push [options] TEMPLATE This will not initiate any builds, it will only update the templates used for builds. + The configuration about what is pushed is configured within the + template's "push" section. + Options: + -create Create the build configuration if it doesn't exist. + -token= Access token to use to upload. If blank, the TODO environmental variable will be used. ` @@ -141,6 +154,39 @@ func (*PushCommand) Synopsis() string { return "push template files to a Packer build service" } +func (c *PushCommand) create(name string, create bool) error { + if c.uploadFn != nil { + return nil + } + + // Separate the slug into the user and name components + user, name, err := harmony.ParseSlug(name) + if err != nil { + return fmt.Errorf("Malformed push name: %s", err) + } + + // Check if it exists. If so, we're done. + if _, err := c.client.BuildConfig(user, name); err == nil { + return nil + } else if err != harmony.ErrNotFound { + return err + } + + // Otherwise, show an error if we're not creating. + if !create { + return fmt.Errorf( + "Push target doesn't exist: %s. Either create this online via\n" + + "the website or pass the -create flag.") + } + + // Create it + if err := c.client.CreateBuildConfig(user, name); err != nil { + return err + } + + return nil +} + func (c *PushCommand) upload( r io.Reader, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { if c.uploadFn != nil { From 6bd8c6ca79963e22e96a85cf6f03a1b9c101fcf5 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 28 Oct 2014 20:06:19 -0700 Subject: [PATCH 26/86] command/push: support custom endpoints --- command/push.go | 10 +++++++++- packer/template.go | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/command/push.go b/command/push.go index 9e7c735bb..4e315f36c 100644 --- a/command/push.go +++ b/command/push.go @@ -58,8 +58,16 @@ func (c *PushCommand) Run(args []string) int { } // Build our client - c.client = harmony.DefaultClient() defer func() { c.client = nil }() + c.client = harmony.DefaultClient() + if tpl.Push.Address != "" { + c.client, err = harmony.NewClient(tpl.Push.Address) + if err != nil { + c.Ui.Error(fmt.Sprintf( + "Error setting up API client: %s", err)) + return 1 + } + } // Build the archiving options var opts archive.ArchiveOpts diff --git a/packer/template.go b/packer/template.go index 562d23edd..0c0bd9bf0 100644 --- a/packer/template.go +++ b/packer/template.go @@ -45,6 +45,7 @@ type Template struct { // PushConfig is the configuration structure for the push settings. type PushConfig struct { Name string + Address string BaseDir string Include []string Exclude []string From c1fbc473a20d1c4e40d6e34985db36b242eb56e0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 6 Nov 2014 08:43:31 -0800 Subject: [PATCH 27/86] command/push: propagate token properly --- command/push.go | 10 +++++++--- packer/template.go | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/command/push.go b/command/push.go index 4e315f36c..8d85a9c99 100644 --- a/command/push.go +++ b/command/push.go @@ -57,6 +57,11 @@ func (c *PushCommand) Run(args []string) int { return 1 } + // Determine our token + if token == "" { + token = tpl.Push.Token + } + // Build our client defer func() { c.client = nil }() c.client = harmony.DefaultClient() @@ -68,6 +73,7 @@ func (c *PushCommand) Run(args []string) int { return 1 } } + c.client.Token = token // Build the archiving options var opts archive.ArchiveOpts @@ -92,7 +98,6 @@ func (c *PushCommand) Run(args []string) int { // Build the upload options var uploadOpts uploadOpts uploadOpts.Slug = tpl.Push.Name - uploadOpts.Token = token uploadOpts.Builds = make(map[string]string) for _, b := range tpl.Builders { uploadOpts.Builds[b.Name] = b.Type @@ -184,7 +189,7 @@ func (c *PushCommand) create(name string, create bool) error { if !create { return fmt.Errorf( "Push target doesn't exist: %s. Either create this online via\n" + - "the website or pass the -create flag.") + "the website or pass the -create flag.", name) } // Create it @@ -244,6 +249,5 @@ func (c *PushCommand) upload( type uploadOpts struct { URL string Slug string - Token string Builds map[string]string } diff --git a/packer/template.go b/packer/template.go index 0c0bd9bf0..a65dbcc90 100644 --- a/packer/template.go +++ b/packer/template.go @@ -49,6 +49,7 @@ type PushConfig struct { BaseDir string Include []string Exclude []string + Token string VCS bool } From d08815fc493ec8eb613d8c97dac046903680e9be Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 1 Dec 2014 15:20:10 -0800 Subject: [PATCH 28/86] command/push: works --- command/push.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/command/push.go b/command/push.go index 8d85a9c99..5aead1f89 100644 --- a/command/push.go +++ b/command/push.go @@ -7,8 +7,8 @@ import ( "path/filepath" "strings" - "github.com/hashicorp/harmony-go" - "github.com/hashicorp/harmony-go/archive" + "github.com/hashicorp/atlas-go/v1" + "github.com/hashicorp/atlas-go/archive" "github.com/mitchellh/packer/packer" ) @@ -18,7 +18,7 @@ const archiveTemplateEntry = ".packer-template" type PushCommand struct { Meta - client *harmony.Client + client *atlas.Client // For tests: uploadFn func(io.Reader, *uploadOpts) (<-chan struct{}, <-chan error, error) @@ -64,9 +64,9 @@ func (c *PushCommand) Run(args []string) int { // Build our client defer func() { c.client = nil }() - c.client = harmony.DefaultClient() + c.client = atlas.DefaultClient() if tpl.Push.Address != "" { - c.client, err = harmony.NewClient(tpl.Push.Address) + c.client, err = atlas.NewClient(tpl.Push.Address) if err != nil { c.Ui.Error(fmt.Sprintf( "Error setting up API client: %s", err)) @@ -110,7 +110,7 @@ func (c *PushCommand) Run(args []string) int { } // Start the archiving process - r, archiveErrCh, err := archive.Archive(path, &opts) + r, err := archive.CreateArchive(path, &opts) if err != nil { c.Ui.Error(fmt.Sprintf("Error archiving: %s", err)) return 1 @@ -126,8 +126,6 @@ func (c *PushCommand) Run(args []string) int { err = nil select { - case err = <-archiveErrCh: - err = fmt.Errorf("Error archiving: %s", err) case err = <-uploadErrCh: err = fmt.Errorf("Error uploading: %s", err) case <-doneCh: @@ -138,6 +136,7 @@ func (c *PushCommand) Run(args []string) int { return 1 } + c.Ui.Output(fmt.Sprintf("Push successful to '%s'", tpl.Push.Name)) return 0 } @@ -173,7 +172,7 @@ func (c *PushCommand) create(name string, create bool) error { } // Separate the slug into the user and name components - user, name, err := harmony.ParseSlug(name) + user, name, err := atlas.ParseSlug(name) if err != nil { return fmt.Errorf("Malformed push name: %s", err) } @@ -181,7 +180,7 @@ func (c *PushCommand) create(name string, create bool) error { // Check if it exists. If so, we're done. if _, err := c.client.BuildConfig(user, name); err == nil { return nil - } else if err != harmony.ErrNotFound { + } else if err != atlas.ErrNotFound { return err } @@ -201,13 +200,13 @@ func (c *PushCommand) create(name string, create bool) error { } func (c *PushCommand) upload( - r io.Reader, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { + r *archive.Archive, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { if c.uploadFn != nil { return c.uploadFn(r, opts) } // Separate the slug into the user and name components - user, name, err := harmony.ParseSlug(opts.Slug) + user, name, err := atlas.ParseSlug(opts.Slug) if err != nil { return nil, nil, fmt.Errorf("upload: %s", err) } @@ -219,13 +218,13 @@ func (c *PushCommand) upload( } // Build the version to send up - version := harmony.BuildConfigVersion{ + version := atlas.BuildConfigVersion{ User: bc.User, Name: bc.Name, - Builds: make([]harmony.BuildConfigBuild, 0, len(opts.Builds)), + Builds: make([]atlas.BuildConfigBuild, 0, len(opts.Builds)), } for name, t := range opts.Builds { - version.Builds = append(version.Builds, harmony.BuildConfigBuild{ + version.Builds = append(version.Builds, atlas.BuildConfigBuild{ Name: name, Type: t, }) @@ -234,7 +233,7 @@ func (c *PushCommand) upload( // Start the upload doneCh, errCh := make(chan struct{}), make(chan error) go func() { - err := c.client.UploadBuildConfigVersion(&version, r) + err := c.client.UploadBuildConfigVersion(&version, r, r.Size) if err != nil { errCh <- err return From f76116e4a38a0a8adec4cf17c1981d0d13de2f4f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 1 Dec 2014 16:54:12 -0800 Subject: [PATCH 29/86] command/push: if a token isn't given, don't set it --- command/push.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/command/push.go b/command/push.go index 5aead1f89..603ef85dc 100644 --- a/command/push.go +++ b/command/push.go @@ -73,7 +73,9 @@ func (c *PushCommand) Run(args []string) int { return 1 } } - c.client.Token = token + if token != "" { + c.client.Token = token + } // Build the archiving options var opts archive.ArchiveOpts From 73c5192b352b053a97159f9c10b15703d0df00df Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 1 Dec 2014 19:49:55 -0800 Subject: [PATCH 30/86] command/push: add base_dir setting --- command/push.go | 25 +++++++++++++++++++++---- packer/template.go | 2 +- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/command/push.go b/command/push.go index 603ef85dc..f7c4f5ebe 100644 --- a/command/push.go +++ b/command/push.go @@ -86,15 +86,32 @@ func (c *PushCommand) Run(args []string) int { archiveTemplateEntry: args[0], } - // Determine the path we're archiving + // Determine the path we're archiving. This logic is a bit complicated + // as there are three possibilities: + // + // 1.) BaseDir is an absolute path, just use that. + // + // 2.) BaseDir is empty, so we use the directory of the template. + // + // 3.) BaseDir is relative, so we use the path relative to the directory + // of the template. + // path := tpl.Push.BaseDir - if path == "" { - path, err = filepath.Abs(args[0]) + if path == "" || !filepath.IsAbs(path) { + tplPath, err := filepath.Abs(args[0]) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error determining path to archive: %s", err)) + return 1 + } + tplPath = filepath.Dir(tplPath) + if path != "" { + tplPath = filepath.Join(tplPath, path) + } + path, err = filepath.Abs(tplPath) if err != nil { c.Ui.Error(fmt.Sprintf("Error determining path to archive: %s", err)) return 1 } - path = filepath.Dir(path) } // Build the upload options diff --git a/packer/template.go b/packer/template.go index a65dbcc90..5cbef4fac 100644 --- a/packer/template.go +++ b/packer/template.go @@ -46,7 +46,7 @@ type Template struct { type PushConfig struct { Name string Address string - BaseDir string + BaseDir string `mapstructure:"base_dir"` Include []string Exclude []string Token string From 2dfc5d3e71993dc37b68df842ca0de072c40f5ca Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 2 Dec 2014 18:22:26 -0800 Subject: [PATCH 31/86] builder/amazon: inject special metadata for atlas artifacts --- builder/amazon/common/artifact.go | 24 ++++++++++++++++++++---- builder/amazon/common/artifact_test.go | 22 +++++++++++++++++++++- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/builder/amazon/common/artifact.go b/builder/amazon/common/artifact.go index 8831d313b..8d6265508 100644 --- a/builder/amazon/common/artifact.go +++ b/builder/amazon/common/artifact.go @@ -2,12 +2,13 @@ package common import ( "fmt" - "github.com/mitchellh/goamz/aws" - "github.com/mitchellh/goamz/ec2" - "github.com/mitchellh/packer/packer" "log" "sort" "strings" + + "github.com/mitchellh/goamz/aws" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/packer/packer" ) // Artifact is an artifact implementation that contains built AMIs. @@ -53,7 +54,12 @@ func (a *Artifact) String() string { } func (a *Artifact) State(name string) interface{} { - return nil + switch name { + case "atlas.artifact.metadata": + return a.stateAtlasMetadata() + default: + return nil + } } func (a *Artifact) Destroy() error { @@ -79,3 +85,13 @@ func (a *Artifact) Destroy() error { return nil } + +func (a *Artifact) stateAtlasMetadata() interface{} { + metadata := make(map[string]string) + for region, imageId := range a.Amis { + k := fmt.Sprintf("region.%s", region) + metadata[k] = imageId + } + + return metadata +} diff --git a/builder/amazon/common/artifact_test.go b/builder/amazon/common/artifact_test.go index ccca4fa0a..46dfedf3f 100644 --- a/builder/amazon/common/artifact_test.go +++ b/builder/amazon/common/artifact_test.go @@ -1,8 +1,10 @@ package common import ( - "github.com/mitchellh/packer/packer" + "reflect" "testing" + + "github.com/mitchellh/packer/packer" ) func TestArtifact_Impl(t *testing.T) { @@ -26,6 +28,24 @@ func TestArtifactId(t *testing.T) { } } +func TestArtifactState_atlasMetadata(t *testing.T) { + a := &Artifact{ + Amis: map[string]string{ + "east": "foo", + "west": "bar", + }, + } + + actual := a.State("atlas.artifact.metadata") + expected := map[string]string{ + "region.east": "foo", + "region.west": "bar", + } + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("bad: %#v", actual) + } +} + func TestArtifactString(t *testing.T) { expected := `AMIs were created: From c9504d50ba5bdf124ee90ba450c71f5200e8821e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 3 Dec 2014 10:01:00 -0800 Subject: [PATCH 32/86] command/push: send artifact true/false if we're post-processing --- command/push.go | 49 +++++++++++++++---- command/push_test.go | 43 +++++++++++++++- .../test-fixtures/push-builds/template.json | 15 ++++++ 3 files changed, 96 insertions(+), 11 deletions(-) create mode 100644 command/test-fixtures/push-builds/template.json diff --git a/command/push.go b/command/push.go index f7c4f5ebe..c59c6cab9 100644 --- a/command/push.go +++ b/command/push.go @@ -7,8 +7,8 @@ import ( "path/filepath" "strings" - "github.com/hashicorp/atlas-go/v1" "github.com/hashicorp/atlas-go/archive" + "github.com/hashicorp/atlas-go/v1" "github.com/mitchellh/packer/packer" ) @@ -21,9 +21,14 @@ type PushCommand struct { client *atlas.Client // For tests: - uploadFn func(io.Reader, *uploadOpts) (<-chan struct{}, <-chan error, error) + uploadFn pushUploadFn } +// pushUploadFn is the callback type used for tests to stub out the uploading +// logic of the push command. +type pushUploadFn func( + io.Reader, *uploadOpts) (<-chan struct{}, <-chan error, error) + func (c *PushCommand) Run(args []string) int { var create bool var token string @@ -114,12 +119,32 @@ func (c *PushCommand) Run(args []string) int { } } + // Find the Atlas post-processors, if possible + var atlasPPs []packer.RawPostProcessorConfig + for _, list := range tpl.PostProcessors { + for _, pp := range list { + if pp.Type == "atlas" { + atlasPPs = append(atlasPPs, pp) + } + } + } + // Build the upload options var uploadOpts uploadOpts uploadOpts.Slug = tpl.Push.Name - uploadOpts.Builds = make(map[string]string) + uploadOpts.Builds = make(map[string]*uploadBuildInfo) for _, b := range tpl.Builders { - uploadOpts.Builds[b.Name] = b.Type + info := &uploadBuildInfo{Type: b.Type} + + // Determine if we're artifacting this build + for _, pp := range atlasPPs { + if !pp.Skip(b.Name) { + info.Artifact = true + break + } + } + + uploadOpts.Builds[b.Name] = info } // Create the build config if it doesn't currently exist. @@ -206,7 +231,7 @@ func (c *PushCommand) create(name string, create bool) error { // Otherwise, show an error if we're not creating. if !create { return fmt.Errorf( - "Push target doesn't exist: %s. Either create this online via\n" + + "Push target doesn't exist: %s. Either create this online via\n"+ "the website or pass the -create flag.", name) } @@ -242,10 +267,11 @@ func (c *PushCommand) upload( Name: bc.Name, Builds: make([]atlas.BuildConfigBuild, 0, len(opts.Builds)), } - for name, t := range opts.Builds { + for name, info := range opts.Builds { version.Builds = append(version.Builds, atlas.BuildConfigBuild{ - Name: name, - Type: t, + Name: name, + Type: info.Type, + Artifact: info.Artifact, }) } @@ -267,5 +293,10 @@ func (c *PushCommand) upload( type uploadOpts struct { URL string Slug string - Builds map[string]string + Builds map[string]*uploadBuildInfo +} + +type uploadBuildInfo struct { + Type string + Artifact bool } diff --git a/command/push_test.go b/command/push_test.go index 6ea1ec326..36e6d11c0 100644 --- a/command/push_test.go +++ b/command/push_test.go @@ -1,10 +1,10 @@ package command import ( - "fmt" "archive/tar" "bytes" "compress/gzip" + "fmt" "io" "path/filepath" "reflect" @@ -59,7 +59,46 @@ func TestPush(t *testing.T) { t.Fatalf("bad: %#v", actual) } - expectedBuilds := map[string]string{"dummy": "dummy"} + expectedBuilds := map[string]*uploadBuildInfo{ + "dummy": &uploadBuildInfo{ + Type: "dummy", + }, + } + if !reflect.DeepEqual(actualOpts.Builds, expectedBuilds) { + t.Fatalf("bad: %#v", actualOpts.Builds) + } +} + +func TestPush_builds(t *testing.T) { + var actualOpts *uploadOpts + uploadFn := func( + r io.Reader, opts *uploadOpts) (<-chan struct{}, <-chan error, error) { + actualOpts = opts + + doneCh := make(chan struct{}) + close(doneCh) + return doneCh, nil, nil + } + + c := &PushCommand{ + Meta: testMeta(t), + uploadFn: uploadFn, + } + + args := []string{filepath.Join(testFixture("push-builds"), "template.json")} + if code := c.Run(args); code != 0 { + fatalCommand(t, c.Meta) + } + + expectedBuilds := map[string]*uploadBuildInfo{ + "dummy": &uploadBuildInfo{ + Type: "dummy", + Artifact: true, + }, + "foo": &uploadBuildInfo{ + Type: "dummy", + }, + } if !reflect.DeepEqual(actualOpts.Builds, expectedBuilds) { t.Fatalf("bad: %#v", actualOpts.Builds) } diff --git a/command/test-fixtures/push-builds/template.json b/command/test-fixtures/push-builds/template.json new file mode 100644 index 000000000..35226ff1e --- /dev/null +++ b/command/test-fixtures/push-builds/template.json @@ -0,0 +1,15 @@ +{ + "builders": [ + {"type": "dummy"}, + {"type": "dummy", "name": "foo"} + ], + + "post-processors": [{ + "type": "atlas", + "only": ["dummy"] + }], + + "push": { + "name": "foo/bar" + } +} From cc61a7dfb87e8aefa9e35f8e9169a6076bcf15d5 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 3 Dec 2014 14:04:01 -0600 Subject: [PATCH 33/86] command/push: warn for bad builds --- command/push.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/command/push.go b/command/push.go index c59c6cab9..edabe8297 100644 --- a/command/push.go +++ b/command/push.go @@ -147,6 +147,26 @@ func (c *PushCommand) Run(args []string) int { uploadOpts.Builds[b.Name] = info } + // Warn about builds not having post-processors. + var badBuilds []string + for name, b := range uploadOpts.Builds { + if b.Artifact { + continue + } + + badBuilds = append(badBuilds, name) + } + if len(badBuilds) > 0 { + c.Ui.Error(fmt.Sprintf( + "Warning! One or more of the builds in this template does not\n" + + "have an Atlas post-processor. Artifacts from this template will\n" + + "not appear in the Atlas artifact registry.\n\n" + + "This is just a warning. Atlas will still build your template\n" + + "and assume other post-processors are sending the artifacts where\n" + + "they need to go.\n\n" + + "Builds: %s\n\n", strings.Join(badBuilds, ", "))) + } + // Create the build config if it doesn't currently exist. if err := c.create(uploadOpts.Slug, create); err != nil { c.Ui.Error(err.Error()) From 80c93bfea0d57a05e52902d40cc16cf6c7f5b7f5 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 3 Dec 2014 14:15:48 -0600 Subject: [PATCH 34/86] command/push: ctrl-C should work properly --- command/push.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/command/push.go b/command/push.go index edabe8297..238e5e088 100644 --- a/command/push.go +++ b/command/push.go @@ -4,6 +4,8 @@ import ( "flag" "fmt" "io" + "os" + "os/signal" "path/filepath" "strings" @@ -158,13 +160,13 @@ func (c *PushCommand) Run(args []string) int { } if len(badBuilds) > 0 { c.Ui.Error(fmt.Sprintf( - "Warning! One or more of the builds in this template does not\n" + - "have an Atlas post-processor. Artifacts from this template will\n" + - "not appear in the Atlas artifact registry.\n\n" + - "This is just a warning. Atlas will still build your template\n" + - "and assume other post-processors are sending the artifacts where\n" + - "they need to go.\n\n" + - "Builds: %s\n\n", strings.Join(badBuilds, ", "))) + "Warning! One or more of the builds in this template does not\n"+ + "have an Atlas post-processor. Artifacts from this template will\n"+ + "not appear in the Atlas artifact registry.\n\n"+ + "This is just a warning. Atlas will still build your template\n"+ + "and assume other post-processors are sending the artifacts where\n"+ + "they need to go.\n\n"+ + "Builds: %s\n\n", strings.Join(badBuilds, ", "))) } // Create the build config if it doesn't currently exist. @@ -188,10 +190,17 @@ func (c *PushCommand) Run(args []string) int { return 1 } + // Make a ctrl-C channel + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, os.Interrupt) + defer signal.Stop(sigCh) + err = nil select { case err = <-uploadErrCh: err = fmt.Errorf("Error uploading: %s", err) + case <-sigCh: + err = fmt.Errorf("Push cancelled from Ctrl-C") case <-doneCh: } @@ -317,6 +326,6 @@ type uploadOpts struct { } type uploadBuildInfo struct { - Type string - Artifact bool + Type string + Artifact bool } From f02000c85dd8f64efa79b4146afdf6ac1b65b22d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 7 Dec 2014 15:21:57 -0600 Subject: [PATCH 35/86] command/push: update help text --- command/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/push.go b/command/push.go index 238e5e088..42c1bc465 100644 --- a/command/push.go +++ b/command/push.go @@ -229,7 +229,7 @@ Options: -create Create the build configuration if it doesn't exist. -token= Access token to use to upload. If blank, the - TODO environmental variable will be used. + ATLAS_TOKEN environmental variable will be used. ` return strings.TrimSpace(helpText) From 6f66afcee08048808fd08ff01734ba59a6f39803 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Dec 2014 14:55:17 -0800 Subject: [PATCH 36/86] website: docs for push --- .../docs/command-line/push.html.markdown | 33 ++++++++++ .../source/docs/templates/push.html.markdown | 65 +++++++++++++++++++ website/source/layouts/docs.erb | 2 + 3 files changed, 100 insertions(+) create mode 100644 website/source/docs/command-line/push.html.markdown create mode 100644 website/source/docs/templates/push.html.markdown diff --git a/website/source/docs/command-line/push.html.markdown b/website/source/docs/command-line/push.html.markdown new file mode 100644 index 000000000..9717f1142 --- /dev/null +++ b/website/source/docs/command-line/push.html.markdown @@ -0,0 +1,33 @@ +--- +layout: "docs" +page_title: "Push - Command-Line" +description: |- + The `packer push` Packer command takes a template and pushes it to a build service that will automatically build this Packer template. +--- + +# Command-Line: Build + +The `packer push` Packer command takes a template and pushes it to a build +service. The build service will automatically build your Packer template and +expose the artifacts. + +This command currently only sends templates to +[Atlas](https://atlas.hashicorp.com) by HashiCorp, but the command will +be pluggable in the future with alternate implementations. + +External build services such as Atlas make it easy to iterate on Packer +templates, especially when the builder you're running may not be easily +accessable (such as developing `qemu` builders on Mac or Windows). + +For the `push` command to work, the +[push configuration](/docs/templates/push.html) +must be completed within the template. + +## Options + +* `-create=true` - If the build configuration matching the name of the push + doesn't exist, it will be created if this is true. This defaults to true. + +* `-token=FOO` - An access token for authenticating the push. This can also + be specified within the push configuration in the template. By setting this + in the template, you can take advantage of user variables. diff --git a/website/source/docs/templates/push.html.markdown b/website/source/docs/templates/push.html.markdown new file mode 100644 index 000000000..1f97a3048 --- /dev/null +++ b/website/source/docs/templates/push.html.markdown @@ -0,0 +1,65 @@ +--- +layout: "docs" +page_title: "Templates: Push" +description: |- + Within the template, the push section configures how a template can be + pushed to a remote build service. +--- + +# Templates: Push + +Within the template, the push section configures how a template can be +[pushed](/docs/command-line/push.html) to a remote build service. + +Push configuration is responsible for defining what files are required +to build this template, what the name of build configuration is in the +build service, etc. + +The only build service that Packer can currently push to is +[Atlas](https://atlas.hashicorp.com) by HashiCorp. Support for other build +services will come in the form of plugins in the future. + +Within a template, a push configuration section looks like this: + +```javascript +{ + "push": { + // ... push configuration here + } +} +``` + +## Configuration Reference + +There are many configuration options available for the builder. They are +segmented below into two categories: required and optional parameters. Within +each category, the available configuration keys are alphabetized. + +### Required + +* `name` (string) - Name of the build configuration in the build service. + If this doesn't exist, it will be created (by default). + +### Optional + +* `address` (string) - The address of the build service to use. By default + this is `https://atlas.hashicorp.com`. + +* `base_dir` (string) - The base directory of the files to upload. This + will be the CWD when the build service executes your template. This + path is relative to the template. + +* `include` (array of strings) - Glob patterns to include relative to + the `base_dir`. If this is specified, only files that match the include + pattern are included. + +* `exclude` (array of strings) - Glob patterns to exclude relative to + the `base_dir`. + +* `token` (string) - An access token to use to authenticate to the build + service. For Atlas, you can retrieve this access token in your account + section by clicking your account name in the upper right corner. + +* `vcs` (bool) - If true, Packer will detect your VCS (if there is one) + and only upload the files that are tracked by the VCS. This is useful + for automatically excluding ignored files. This defaults to true. diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index b686bb515..a403ad336 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -13,6 +13,7 @@
  • Build
  • Fix
  • Inspect
    • +
  • Push
  • Validate
  • Machine-Readable Output
    • @@ -23,6 +24,7 @@
  • Builders
  • Provisioners
  • Post-Processors
    • +
  • Push
  • Configuration Templates
  • User Variables
  • Veewee-to-Packer
    • From f3c1132f23db40946854f6517dad7dccb87729bb Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Dec 2014 15:00:03 -0800 Subject: [PATCH 37/86] post-processor/atlas: make it --- plugin/post-processor-atlas/main.go | 15 ++ plugin/post-processor-atlas/main_test.go | 1 + post-processor/atlas/artifact.go | 37 +++ post-processor/atlas/post-processor.go | 258 ++++++++++++++++++++ post-processor/atlas/post-processor_test.go | 136 +++++++++++ post-processor/atlas/util.go | 47 ++++ post-processor/atlas/util_test.go | 36 +++ 7 files changed, 530 insertions(+) create mode 100644 plugin/post-processor-atlas/main.go create mode 100644 plugin/post-processor-atlas/main_test.go create mode 100644 post-processor/atlas/artifact.go create mode 100644 post-processor/atlas/post-processor.go create mode 100644 post-processor/atlas/post-processor_test.go create mode 100644 post-processor/atlas/util.go create mode 100644 post-processor/atlas/util_test.go diff --git a/plugin/post-processor-atlas/main.go b/plugin/post-processor-atlas/main.go new file mode 100644 index 000000000..21bbd13cf --- /dev/null +++ b/plugin/post-processor-atlas/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/mitchellh/packer/packer/plugin" + "github.com/mitchellh/packer/post-processor/atlas" +) + +func main() { + server, err := plugin.Server() + if err != nil { + panic(err) + } + server.RegisterPostProcessor(new(atlas.PostProcessor)) + server.Serve() +} diff --git a/plugin/post-processor-atlas/main_test.go b/plugin/post-processor-atlas/main_test.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/plugin/post-processor-atlas/main_test.go @@ -0,0 +1 @@ +package main diff --git a/post-processor/atlas/artifact.go b/post-processor/atlas/artifact.go new file mode 100644 index 000000000..50c503a9b --- /dev/null +++ b/post-processor/atlas/artifact.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" +) + +const BuilderId = "packer.post-processor.atlas" + +type Artifact struct { + Name string + Type string + Version int +} + +func (*Artifact) BuilderId() string { + return BuilderId +} + +func (a *Artifact) Files() []string { + return nil +} + +func (a *Artifact) Id() string { + return fmt.Sprintf("%s/%s/%d", a.Name, a.Type, a.Version) +} + +func (a *Artifact) String() string { + return fmt.Sprintf("%s/%s (v%d)", a.Name, a.Type, a.Version) +} + +func (*Artifact) State(name string) interface{} { + return nil +} + +func (a *Artifact) Destroy() error { + return nil +} diff --git a/post-processor/atlas/post-processor.go b/post-processor/atlas/post-processor.go new file mode 100644 index 000000000..40c2ce397 --- /dev/null +++ b/post-processor/atlas/post-processor.go @@ -0,0 +1,258 @@ +package main + +import ( + "fmt" + "os" + "strconv" + "strings" + + "github.com/hashicorp/atlas-go/archive" + "github.com/hashicorp/atlas-go/v1" + "github.com/mitchellh/mapstructure" + "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/packer" +) + +const BuildEnvKey = "ATLAS_BUILD_ID" + +// Artifacts can return a string for this state key and the post-processor +// will use automatically use this as the type. The user's value overrides +// this if `artifact_type_override` is set to true. +const ArtifactStateType = "atlas.artifact.type" + +// Artifacts can return a map[string]string for this state key and this +// post-processor will automatically merge it into the metadata for any +// uploaded artifact versions. +const ArtifactStateMetadata = "atlas.artifact.metadata" + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + Artifact string + Type string `mapstructure:"artifact_type"` + TypeOverride bool `mapstructure:"artifact_type_override"` + Metadata map[string]string + + ServerAddr string `mapstructure:"server_address"` + Token string + + // This shouldn't ever be set outside of unit tests. + Test bool `mapstructure:"test"` + + tpl *packer.ConfigTemplate + user, name string + buildId int +} + +type PostProcessor struct { + config Config + client *atlas.Client +} + +func (p *PostProcessor) Configure(raws ...interface{}) error { + _, err := common.DecodeConfig(&p.config, raws...) + if err != nil { + return err + } + + p.config.tpl, err = packer.NewConfigTemplate() + if err != nil { + return err + } + p.config.tpl.UserVars = p.config.PackerUserVars + + templates := map[string]*string{ + "artifact": &p.config.Artifact, + "type": &p.config.Type, + "server_address": &p.config.ServerAddr, + "token": &p.config.Token, + } + + errs := new(packer.MultiError) + for key, ptr := range templates { + *ptr, err = p.config.tpl.Process(*ptr, nil) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Error processing %s: %s", key, err)) + } + } + + required := map[string]*string{ + "artifact": &p.config.Artifact, + "artifact_type": &p.config.Type, + } + + for key, ptr := range required { + if *ptr == "" { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("%s must be set", key)) + } + } + + if len(errs.Errors) > 0 { + return errs + } + + p.config.user, p.config.name, err = atlas.ParseSlug(p.config.Artifact) + if err != nil { + return err + } + + // If we have a build ID, save it + if v := os.Getenv(BuildEnvKey); v != "" { + raw, err := strconv.ParseInt(v, 0, 0) + if err != nil { + return fmt.Errorf( + "Error parsing build ID: %s", err) + } + + p.config.buildId = int(raw) + } + + // Build the client + p.client = atlas.DefaultClient() + if p.config.ServerAddr != "" { + p.client, err = atlas.NewClient(p.config.ServerAddr) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Error initializing client: %s", err)) + return errs + } + } + if p.config.Token != "" { + p.client.Token = p.config.Token + } + + if !p.config.Test { + // Verify the client + if err := p.client.Verify(); err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Error initializing client: %s", err)) + return errs + } + } + + return nil +} + +func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { + if _, err := p.client.Artifact(p.config.user, p.config.name); err != nil { + if err != atlas.ErrNotFound { + return nil, false, fmt.Errorf( + "Error finding artifact: %s", err) + } + + // Artifact doesn't exist, create it + ui.Message(fmt.Sprintf("Creating artifact: %s", p.config.Artifact)) + _, err = p.client.CreateArtifact(p.config.user, p.config.name) + if err != nil { + return nil, false, fmt.Errorf( + "Error creating artifact: %s", err) + } + } + + opts := &atlas.UploadArtifactOpts{ + User: p.config.user, + Name: p.config.name, + Type: p.config.Type, + ID: artifact.Id(), + Metadata: p.metadata(artifact), + BuildId: p.config.buildId, + } + + if fs := artifact.Files(); len(fs) > 0 { + var archiveOpts archive.ArchiveOpts + + // We have files. We want to compress/upload them. If we have just + // one file, then we use it as-is. Otherwise, we compress all of + // them into a single file. + var path string + if len(fs) == 1 { + path = fs[0] + } else { + path = longestCommonPrefix(fs) + if path == "" { + return nil, false, fmt.Errorf( + "No common prefix for achiving files: %v", fs) + } + + // Modify the archive options to only include the files + // that are in our file list. + include := make([]string, 0, len(fs)) + for i, f := range fs { + include[i] = strings.Replace(f, path, "", 1) + } + archiveOpts.Include = include + } + + r, err := archive.CreateArchive(path, &archiveOpts) + if err != nil { + return nil, false, fmt.Errorf( + "Error archiving artifact: %s", err) + } + defer r.Close() + + opts.File = r + opts.FileSize = r.Size + } + + ui.Message("Uploading artifact version...") + var av *atlas.ArtifactVersion + doneCh := make(chan struct{}) + errCh := make(chan error, 1) + go func() { + var err error + av, err = p.client.UploadArtifact(opts) + if err != nil { + errCh <- err + return + } + close(doneCh) + }() + + select { + case err := <-errCh: + return nil, false, fmt.Errorf("Error uploading: %s", err) + case <-doneCh: + } + + return &Artifact{ + Name: p.config.Artifact, + Type: p.config.Type, + Version: av.Version, + }, true, nil +} + +func (p *PostProcessor) metadata(artifact packer.Artifact) map[string]string { + var metadata map[string]string + metadataRaw := artifact.State(ArtifactStateMetadata) + if metadataRaw != nil { + if err := mapstructure.Decode(metadataRaw, &metadata); err != nil { + panic(err) + } + } + + if p.config.Metadata != nil { + // If we have no extra metadata, just return as-is + if metadata == nil { + return p.config.Metadata + } + + // Merge the metadata + for k, v := range p.config.Metadata { + metadata[k] = v + } + } + + return metadata +} + +func (p *PostProcessor) artifactType(artifact packer.Artifact) string { + if !p.config.TypeOverride { + if v := artifact.State(ArtifactStateType); v != nil { + return v.(string) + } + } + + return p.config.Type +} diff --git a/post-processor/atlas/post-processor_test.go b/post-processor/atlas/post-processor_test.go new file mode 100644 index 000000000..62fa0e07a --- /dev/null +++ b/post-processor/atlas/post-processor_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "os" + "reflect" + "testing" + + "github.com/mitchellh/packer/packer" +) + +func TestPostProcessorConfigure(t *testing.T) { + var p PostProcessor + if err := p.Configure(validDefaults()); err != nil { + t.Fatalf("err: %s", err) + } + + if p.client == nil { + t.Fatal("should have client") + } + if p.client.Token != "" { + t.Fatal("should not have token") + } +} + +func TestPostProcessorConfigure_buildId(t *testing.T) { + defer os.Setenv(BuildEnvKey, os.Getenv(BuildEnvKey)) + os.Setenv(BuildEnvKey, "5") + + var p PostProcessor + if err := p.Configure(validDefaults()); err != nil { + t.Fatalf("err: %s", err) + } + + if p.config.buildId != 5 { + t.Fatalf("bad: %#v", p.config.buildId) + } +} + +func TestPostProcessorMetadata(t *testing.T) { + var p PostProcessor + if err := p.Configure(validDefaults()); err != nil { + t.Fatalf("err: %s", err) + } + + artifact := new(packer.MockArtifact) + metadata := p.metadata(artifact) + if len(metadata) > 0 { + t.Fatalf("bad: %#v", metadata) + } +} + +func TestPostProcessorMetadata_artifact(t *testing.T) { + config := validDefaults() + config["metadata"] = map[string]string{ + "foo": "bar", + } + + var p PostProcessor + if err := p.Configure(config); err != nil { + t.Fatalf("err: %s", err) + } + + artifact := new(packer.MockArtifact) + artifact.StateValues = map[string]interface{}{ + ArtifactStateMetadata: map[interface{}]interface{}{ + "bar": "baz", + }, + } + + metadata := p.metadata(artifact) + expected := map[string]string{ + "foo": "bar", + "bar": "baz", + } + if !reflect.DeepEqual(metadata, expected) { + t.Fatalf("bad: %#v", metadata) + } +} + +func TestPostProcessorMetadata_config(t *testing.T) { + config := validDefaults() + config["metadata"] = map[string]string{ + "foo": "bar", + } + + var p PostProcessor + if err := p.Configure(config); err != nil { + t.Fatalf("err: %s", err) + } + + artifact := new(packer.MockArtifact) + metadata := p.metadata(artifact) + expected := map[string]string{ + "foo": "bar", + } + if !reflect.DeepEqual(metadata, expected) { + t.Fatalf("bad: %#v", metadata) + } +} + +func TestPostProcessorType(t *testing.T) { + var p PostProcessor + if err := p.Configure(validDefaults()); err != nil { + t.Fatalf("err: %s", err) + } + + artifact := new(packer.MockArtifact) + actual := p.artifactType(artifact) + if actual != "foo" { + t.Fatalf("bad: %#v", actual) + } +} + +func TestPostProcessorType_artifact(t *testing.T) { + var p PostProcessor + if err := p.Configure(validDefaults()); err != nil { + t.Fatalf("err: %s", err) + } + + artifact := new(packer.MockArtifact) + artifact.StateValues = map[string]interface{}{ + ArtifactStateType: "bar", + } + actual := p.artifactType(artifact) + if actual != "bar" { + t.Fatalf("bad: %#v", actual) + } +} + +func validDefaults() map[string]interface{} { + return map[string]interface{}{ + "artifact": "mitchellh/test", + "artifact_type": "foo", + "test": true, + } +} diff --git a/post-processor/atlas/util.go b/post-processor/atlas/util.go new file mode 100644 index 000000000..dba2076c7 --- /dev/null +++ b/post-processor/atlas/util.go @@ -0,0 +1,47 @@ +package main + +import ( + "math" + "strings" +) + +// longestCommonPrefix finds the longest common prefix for all the strings +// given as an argument, or returns the empty string if a prefix can't be +// found. +// +// This function just uses brute force instead of a more optimized algorithm. +func longestCommonPrefix(vs []string) string { + // Find the shortest string + var shortest string + length := math.MaxUint32 + for _, v := range vs { + if len(v) < length { + shortest = v + length = len(v) + } + } + + // Now go through and find a prefix to all the strings using this + // short string, which itself must contain the prefix. + for i := len(shortest); i > 0; i-- { + // We only care about prefixes with path seps + if shortest[i-1] != '/' { + continue + } + + bad := false + prefix := shortest[0 : i] + for _, v := range vs { + if !strings.HasPrefix(v, prefix) { + bad = true + break + } + } + + if !bad { + return prefix + } + } + + return "" +} diff --git a/post-processor/atlas/util_test.go b/post-processor/atlas/util_test.go new file mode 100644 index 000000000..181591e84 --- /dev/null +++ b/post-processor/atlas/util_test.go @@ -0,0 +1,36 @@ +package main + +import ( + "testing" +) + +func TestLongestCommonPrefix(t *testing.T) { + cases := []struct { + Input []string + Output string + }{ + { + []string{"foo", "bar"}, + "", + }, + { + []string{"foo", "foobar"}, + "", + }, + { + []string{"foo/", "foo/bar"}, + "foo/", + }, + { + []string{"/foo/", "/bar"}, + "/", + }, + } + + for _, tc := range cases { + actual := longestCommonPrefix(tc.Input) + if actual != tc.Output { + t.Fatalf("bad: %#v\n\n%#v", actual, tc.Input) + } + } +} From 0ffd184be63f4a8acef920a83fc2bc30189a682c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Dec 2014 15:01:46 -0800 Subject: [PATCH 38/86] post-processor/atlas: proper package --- post-processor/atlas/artifact.go | 2 +- post-processor/atlas/post-processor.go | 2 +- post-processor/atlas/post-processor_test.go | 2 +- post-processor/atlas/util.go | 2 +- post-processor/atlas/util_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/post-processor/atlas/artifact.go b/post-processor/atlas/artifact.go index 50c503a9b..0a8a10ac1 100644 --- a/post-processor/atlas/artifact.go +++ b/post-processor/atlas/artifact.go @@ -1,4 +1,4 @@ -package main +package atlas import ( "fmt" diff --git a/post-processor/atlas/post-processor.go b/post-processor/atlas/post-processor.go index 40c2ce397..530c4e5d9 100644 --- a/post-processor/atlas/post-processor.go +++ b/post-processor/atlas/post-processor.go @@ -1,4 +1,4 @@ -package main +package atlas import ( "fmt" diff --git a/post-processor/atlas/post-processor_test.go b/post-processor/atlas/post-processor_test.go index 62fa0e07a..db35b8790 100644 --- a/post-processor/atlas/post-processor_test.go +++ b/post-processor/atlas/post-processor_test.go @@ -1,4 +1,4 @@ -package main +package atlas import ( "os" diff --git a/post-processor/atlas/util.go b/post-processor/atlas/util.go index dba2076c7..7941db563 100644 --- a/post-processor/atlas/util.go +++ b/post-processor/atlas/util.go @@ -1,4 +1,4 @@ -package main +package atlas import ( "math" diff --git a/post-processor/atlas/util_test.go b/post-processor/atlas/util_test.go index 181591e84..9f2535ccb 100644 --- a/post-processor/atlas/util_test.go +++ b/post-processor/atlas/util_test.go @@ -1,4 +1,4 @@ -package main +package atlas import ( "testing" From 59e88555d75753a124d5b49184bff53f0b7caef3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Dec 2014 18:56:41 -0800 Subject: [PATCH 39/86] Update CHANGELOG --- CHANGELOG.md | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4f9c78f9..782fe84b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,33 +1,39 @@ -## 0.8.0 (unreleased) +## 0.7.5 (unreleased) FEATURES: + * **New command: `packer push`**: Push template and files to HashiCorp's + Atlas for building your templates automatically. + * **New post-processor: `atlas`**: Send artifact to HashiCorp's Atlas for + versioning and storing artifacts. These artifacts can then be queried + using the API, Terraform, etc. + IMPROVEMENTS: -* builder/googlecompute: Support for ubuntu-os-cloud project -* builder/googlecompute: Support for OAuth2 to avoid client secrets file -* builder/googlecompute: GCE image from persistant disk instead of tarball -* builder/qemu: Checksum type "none" can be used -* provisioner/chef: Generate a node name if none available -* provisioner/chef: Added ssl_verify_mode configuration + * builder/googlecompute: Support for ubuntu-os-cloud project + * builder/googlecompute: Support for OAuth2 to avoid client secrets file + * builder/googlecompute: GCE image from persistant disk instead of tarball + * builder/qemu: Checksum type "none" can be used + * provisioner/chef: Generate a node name if none available + * provisioner/chef: Added ssl_verify_mode configuration BUG FIXES: -* builder/parallels: Fixed attachment of ISO to cdrom device -* builder/parallels: Fixed boot load ordering -* builder/digitalocean: Fixed decoding of size -* builder/digitalocean: Fixed missing content-type header in request -* builder/digitalocean: Fixed use of private IP -* builder/digitalocean: Fixed the artifact ID generation -* builder/vsphere: Fixed credential escaping -* builder/qemu: Fixed use of CDROM with disk_image -* builder/aws: Fixed IP address for SSH in VPC -* builder/aws: Fixed issue with multiple block devices -* builder/vmware: Upload VMX to ESX5 after editing -* communicator/docker: Fix handling of symlinks during upload -* provisioner/chef: Fixed use of sudo in some cases -* core: Fixed build name interpolation -* postprocessor/vagrant: Fixed check for Vagrantfile template + * builder/parallels: Fixed attachment of ISO to cdrom device + * builder/parallels: Fixed boot load ordering + * builder/digitalocean: Fixed decoding of size + * builder/digitalocean: Fixed missing content-type header in request + * builder/digitalocean: Fixed use of private IP + * builder/digitalocean: Fixed the artifact ID generation + * builder/vsphere: Fixed credential escaping + * builder/qemu: Fixed use of CDROM with disk_image + * builder/aws: Fixed IP address for SSH in VPC + * builder/aws: Fixed issue with multiple block devices + * builder/vmware: Upload VMX to ESX5 after editing + * communicator/docker: Fix handling of symlinks during upload + * provisioner/chef: Fixed use of sudo in some cases + * core: Fixed build name interpolation + * postprocessor/vagrant: Fixed check for Vagrantfile template ## 0.7.2 (October 28, 2014) From 9cd66feeacbd9cb318b72eb5ed59428c5b8c37ac Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Dec 2014 18:57:03 -0800 Subject: [PATCH 40/86] Update CHANGELOG --- CHANGELOG.md | 2 +- version.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 782fe84b0..492e96ea8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 0.7.5 (unreleased) +## 0.7.5 (December 9, 2014) FEATURES: diff --git a/version.go b/version.go index dbfc8da12..f7e44a1d0 100644 --- a/version.go +++ b/version.go @@ -4,9 +4,9 @@ package main var GitCommit string // The main version number that is being run at the moment. -const Version = "0.8.0" +const Version = "0.7.5" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -const VersionPrerelease = "dev" +const VersionPrerelease = "" From ccc09c01673fd5335f6914f701c952164927eb86 Mon Sep 17 00:00:00 2001 From: jenspinney Date: Tue, 9 Dec 2014 22:18:59 -0800 Subject: [PATCH 41/86] Fix documentation for openstack_provider and provider config options. --- .../docs/builders/openstack.html.markdown | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/website/source/docs/builders/openstack.html.markdown b/website/source/docs/builders/openstack.html.markdown index 0e864e4bd..7fc8182eb 100644 --- a/website/source/docs/builders/openstack.html.markdown +++ b/website/source/docs/builders/openstack.html.markdown @@ -38,11 +38,6 @@ each category, the available configuration keys are alphabetized. If not specified, Packer will use the environment variables `SDK_PASSWORD` or `OS_PASSWORD` (in that order), if set. -* `provider` (string) - The provider used to connect to the OpenStack service. - If not specified, Packer will use the environment variable - `SDK_PROVIDER`, if set. - For Rackspace this should be `rackspace-us` or `rackspace-uk`. - * `source_image` (string) - The ID or full URL to the base image to use. This is the image that will be used to launch a new server and provision it. @@ -70,18 +65,19 @@ each category, the available configuration keys are alphabetized. * `networks` (array of strings) - A list of networks by UUID to attach to this instance. -* `openstack_provider` (string) +* `openstack_provider` (string) - A name of a provider that has a slightly + different API model. Currently supported values are "openstack" (default), + and "rackspace". * `project` (string) - The project name to boot the instance into. Some OpenStack installations require this. If not specified, Packer will use the environment variables `SDK_PROJECT` or `OS_TENANT_NAME` (in that order), if set. -* `provider` (string) - A name of a provider that has a slightly - different API model. Currently supported values are "openstack" (default), - and "rackspace". - If not specified, Packer will use the environment variables - `SDK_PROVIDER` or `OS_AUTH_URL` (in that order), if set. +* `provider` (string) - The provider used to connect to the OpenStack service. + If not specified, Packer will use the environment variables `SDK_PROVIDER` + or `OS_AUTH_URL` (in that order), if set. + For Rackspace this should be `rackspace-us` or `rackspace-uk`. * `proxy_url` (string) @@ -124,7 +120,6 @@ Ubuntu 12.04 LTS (Precise Pangolin) on Rackspace OpenStack cloud offering. "api_key": "", "openstack_provider": "rackspace", "provider": "rackspace-us", - "openstack_provider":"rackspace", "region": "DFW", "ssh_username": "root", "image_name": "Test image", From 010c77d05f0e53b034e298f6e0280861d5444a5a Mon Sep 17 00:00:00 2001 From: Ryan Uber Date: Wed, 10 Dec 2014 00:24:27 -0800 Subject: [PATCH 42/86] builder/digitalocean: default to nyc3 --- builder/digitalocean/builder.go | 4 ++-- website/source/docs/builders/digitalocean.html.markdown | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index ad41ee547..cf76f6970 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -21,8 +21,8 @@ import ( const DefaultImage = "ubuntu-12-04-x64" // see https://api.digitalocean.com/regions/?client_id=[client_id]&api_key=[api_key] -// name="New York", id=1 -const DefaultRegion = "nyc1" +// name="New York 3", id=8 +const DefaultRegion = "nyc3" // see https://api.digitalocean.com/sizes/?client_id=[client_id]&api_key=[api_key] // name="512MB", id=66 (the smallest droplet size) diff --git a/website/source/docs/builders/digitalocean.html.markdown b/website/source/docs/builders/digitalocean.html.markdown index fd92b9e51..82bf0c518 100644 --- a/website/source/docs/builders/digitalocean.html.markdown +++ b/website/source/docs/builders/digitalocean.html.markdown @@ -66,7 +66,7 @@ each category, the available configuration keys are alphabetized. * `region` (string) - The name (or slug) of the region to launch the droplet in. Consequently, this is the region where the snapshot will be available. - This defaults to "nyc1", which is the slug for "New York 1". + This defaults to "nyc3", which is the slug for "New York 3". See https://developers.digitalocean.com/regions/ for the accepted region names/slugs. * `region_id` (integer) - The ID of the region to launch the droplet in. Consequently, From 8aac8dc17c0265903edcc72087dbbfbd5749f729 Mon Sep 17 00:00:00 2001 From: Danny Lockard Date: Wed, 10 Dec 2014 11:35:32 -0600 Subject: [PATCH 43/86] Per https://coderwall.com/p/momg4a/packer-io-vsphere-post-processor-without-resource-pools --- website/source/docs/post-processors/vsphere.html.markdown | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/source/docs/post-processors/vsphere.html.markdown b/website/source/docs/post-processors/vsphere.html.markdown index 3ce8ec5f9..d49683a43 100644 --- a/website/source/docs/post-processors/vsphere.html.markdown +++ b/website/source/docs/post-processors/vsphere.html.markdown @@ -31,7 +31,8 @@ Required: * `password` (string) - Password to use to authenticate to the vSphere endpoint. -* `resource_pool` (string) - The resource pool to upload the VM to. +* `resource_pool` (string) - The resource pool to upload the VM to. This can be + " " if you do not have resource pools configured * `username` (string) - The username to use to authenticate to the vSphere endpoint. From ddb966061f88709537c93f16f7c4066ddf2b8adf Mon Sep 17 00:00:00 2001 From: BlackEagle Date: Thu, 11 Dec 2014 13:47:28 +0100 Subject: [PATCH 44/86] atlas post-processor unknown atlas.UploadartifactOpts field BuildId in https://github.com/hashicorp/atlas-go/blob/master/v1/artifact.go we can see it is BuildID and not BuildId Signed-off-by: BlackEagle --- post-processor/atlas/post-processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/post-processor/atlas/post-processor.go b/post-processor/atlas/post-processor.go index 530c4e5d9..f66eab9ab 100644 --- a/post-processor/atlas/post-processor.go +++ b/post-processor/atlas/post-processor.go @@ -157,7 +157,7 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac Type: p.config.Type, ID: artifact.Id(), Metadata: p.metadata(artifact), - BuildId: p.config.buildId, + BuildID: p.config.buildId, } if fs := artifact.Files(); len(fs) > 0 { From 272ef5a6ba0c8d92d68b3688440066e23aba88c5 Mon Sep 17 00:00:00 2001 From: BlackEagle Date: Thu, 11 Dec 2014 14:24:17 +0100 Subject: [PATCH 45/86] atlas post-processor on 32bit uint32 overflows int post-processor/atlas/util.go:16: constant 4294967295 overflows int Signed-off-by: BlackEagle --- post-processor/atlas/util.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/post-processor/atlas/util.go b/post-processor/atlas/util.go index 7941db563..1c0bfa27f 100644 --- a/post-processor/atlas/util.go +++ b/post-processor/atlas/util.go @@ -11,13 +11,14 @@ import ( // // This function just uses brute force instead of a more optimized algorithm. func longestCommonPrefix(vs []string) string { + var length int64 // Find the shortest string var shortest string - length := math.MaxUint32 + length = math.MaxUint32 for _, v := range vs { - if len(v) < length { + if int64(len(v)) < length { shortest = v - length = len(v) + length = int64(len(v)) } } From 1e88d41b6fcc5a7312d84a3abffd79b00939112c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Czeraszkiewicz?= Date: Fri, 12 Dec 2014 02:30:10 +0100 Subject: [PATCH 46/86] Fix broken links in the DigitalOcean documentation --- website/source/docs/builders/digitalocean.html.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/source/docs/builders/digitalocean.html.markdown b/website/source/docs/builders/digitalocean.html.markdown index 82bf0c518..299e88eb1 100644 --- a/website/source/docs/builders/digitalocean.html.markdown +++ b/website/source/docs/builders/digitalocean.html.markdown @@ -67,7 +67,7 @@ each category, the available configuration keys are alphabetized. * `region` (string) - The name (or slug) of the region to launch the droplet in. Consequently, this is the region where the snapshot will be available. This defaults to "nyc3", which is the slug for "New York 3". - See https://developers.digitalocean.com/regions/ for the accepted region names/slugs. + See https://developers.digitalocean.com/v2/#regions for the accepted region names/slugs. * `region_id` (integer) - The ID of the region to launch the droplet in. Consequently, this is the region where the snapshot will be available. @@ -75,7 +75,7 @@ each category, the available configuration keys are alphabetized. * `size` (string) - The name (or slug) of the droplet size to use. This defaults to "512mb", which is the slug for "512MB". - See https://developers.digitalocean.com/sizes/ for the accepted size names/slugs. + See https://developers.digitalocean.com/#sizes for the accepted size names/slugs. * `size_id` (integer) - The ID of the droplet size to use. This setting is deprecated. Use `size` instead. From 8b10e70bc8da826e510d66c16d7933e59d993d3b Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Sat, 13 Dec 2014 19:26:58 +0100 Subject: [PATCH 47/86] fixed packer --version --- main.go | 1 + test/cli.bats | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 093719b3f..2bebafb9e 100644 --- a/main.go +++ b/main.go @@ -166,6 +166,7 @@ func wrappedMain() int { Commands: Commands, HelpFunc: cli.BasicHelpFunc("packer"), HelpWriter: os.Stdout, + Version: Version, } exitCode, err := cli.Run() diff --git a/test/cli.bats b/test/cli.bats index 5eaa3cbd9..0284dd3e3 100755 --- a/test/cli.bats +++ b/test/cli.bats @@ -17,20 +17,20 @@ load test_helper [[ "$output" == *"Packer v"* ]] run packer -v - [ "$status" -eq 0 ] - [[ "$output" == *"Packer v"* ]] + [ "$status" -eq 1 ] + [[ "$output" =~ ([0-9]+\.[0-9]+) ]] run packer --version - [ "$status" -eq 0 ] - [[ "$output" == *"Packer v"* ]] + [ "$status" -eq 1 ] + [[ "$output" =~ ([0-9]+\.[0-9]+) ]] } @test "cli: packer version show help" { run packer version -h [ "$status" -eq 0 ] - [[ "$output" == *"usage: packer version"* ]] + [[ "$output" == *"Packer v"* ]] run packer version --help [ "$status" -eq 0 ] - [[ "$output" == *"usage: packer version"* ]] + [[ "$output" == *"Packer v"* ]] } From 118325f6b6fbb9f7c7a17b069561cfd67e0fb070 Mon Sep 17 00:00:00 2001 From: Emil Hessman Date: Mon, 15 Dec 2014 07:03:33 +0100 Subject: [PATCH 48/86] website/intro/getting-started: clarify that multiple builders of the same type must have a unique name --- .../source/intro/getting-started/parallel-builds.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/intro/getting-started/parallel-builds.html.markdown b/website/source/intro/getting-started/parallel-builds.html.markdown index 72ab4737c..8a125719e 100644 --- a/website/source/intro/getting-started/parallel-builds.html.markdown +++ b/website/source/intro/getting-started/parallel-builds.html.markdown @@ -83,7 +83,7 @@ The entire template should now [look like this](https://gist.github.com/pearkes/ Additional builders are simply added to the `builders` array in the template. This tells Packer to build multiple images. The builder `type` values don't even need to be different! In fact, if you wanted to build multiple AMIs, -you can do that as well. +you can do that as long as you specify a unique `name` for each build. Validate the template with `packer validate`. This is always a good practice. From 0f36a051d5c4d6e33c0d61d45e1bd6a94e1312d0 Mon Sep 17 00:00:00 2001 From: Tim Cinel Date: Tue, 16 Dec 2014 14:11:28 +1100 Subject: [PATCH 49/86] Create temporary copy of symlink before uploading, fixes #1765 --- communicator/ssh/communicator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/communicator/ssh/communicator.go b/communicator/ssh/communicator.go index 245f3ceb0..07fb1eaa2 100644 --- a/communicator/ssh/communicator.go +++ b/communicator/ssh/communicator.go @@ -336,7 +336,7 @@ func scpUploadFile(dst string, src io.Reader, w io.Writer, r *bufio.Reader, fi * var mode os.FileMode var size int64 - if fi != nil { + if fi != nil && (*fi).Mode().IsRegular() { mode = (*fi).Mode().Perm() size = (*fi).Size() } else { From 5a9e5df967ceb223f65fb316e9ebf2902637e1e6 Mon Sep 17 00:00:00 2001 From: Neeki Patel Date: Tue, 16 Dec 2014 09:27:30 -0500 Subject: [PATCH 50/86] Allow manifest_file to be a directory Puppet is removing support for imports and node inherits. This change will support directories as manifests https://docs.puppetlabs.com/puppet/latest/reference/dirs_manifest.html https://docs.puppetlabs.com/puppet/latest/reference/lang_import.html --- provisioner/puppet-masterless/provisioner.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/provisioner/puppet-masterless/provisioner.go b/provisioner/puppet-masterless/provisioner.go index 81ab34afa..6cd62c28b 100644 --- a/provisioner/puppet-masterless/provisioner.go +++ b/provisioner/puppet-masterless/provisioner.go @@ -180,9 +180,6 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { if err != nil { errs = packer.MultiErrorAppend(errs, fmt.Errorf("manifest_file is invalid: %s", err)) - } else if info.IsDir() { - errs = packer.MultiErrorAppend(errs, - fmt.Errorf("manifest_file must point to a file")) } } From 027b43ea05171ac761fa0c913945a95bf02893ae Mon Sep 17 00:00:00 2001 From: Neeki Patel Date: Tue, 16 Dec 2014 09:49:57 -0500 Subject: [PATCH 51/86] Fixed info declared and not used --- provisioner/puppet-masterless/provisioner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioner/puppet-masterless/provisioner.go b/provisioner/puppet-masterless/provisioner.go index 6cd62c28b..307ecce38 100644 --- a/provisioner/puppet-masterless/provisioner.go +++ b/provisioner/puppet-masterless/provisioner.go @@ -176,7 +176,7 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { errs = packer.MultiErrorAppend(errs, fmt.Errorf("A manifest_file must be specified.")) } else { - info, err := os.Stat(p.config.ManifestFile) + _, err := os.Stat(p.config.ManifestFile) if err != nil { errs = packer.MultiErrorAppend(errs, fmt.Errorf("manifest_file is invalid: %s", err)) From bce1a5d9d7e9795e7dc382d50386f0d3bf05afec Mon Sep 17 00:00:00 2001 From: Matthew Barrington Date: Tue, 16 Dec 2014 17:53:12 +0000 Subject: [PATCH 52/86] Updated the spot price docs --- website/source/docs/builders/amazon-ebs.html.markdown | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index a52501986..4700ade55 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -118,11 +118,11 @@ each category, the available configuration keys are alphabetized. described above. Note that if this is specified, you must omit the `security_group_id`. -* `spot_price` (string) - The maximum hourly price to launch a spot instance - to create the AMI. It is a type of instances that EC2 starts when the maximum - price that you specify exceeds the current spot price. Spot price will be - updated based on available spot instance capacity and current spot Instance - requests. It may save you some costs. You can set this to "auto" for +* `spot_price` (string) - The maximum hourly price to pay for a spot instance + to create the AMI. Spot instances are a type of instance that EC2 starts when + the current spot price is less than the maximum price you specify. Spot price + will be updated based on available spot instance capacity and current spot + instance requests. It may save you some costs. You can set this to "auto" for Packer to automatically discover the best spot price. * `spot_price_auto_product` (string) - Required if `spot_price` is set to From d414ed0d49ae1da32079a6da7bbd041b7cd0783a Mon Sep 17 00:00:00 2001 From: Joseph Anthony Pasquale Holsten Date: Tue, 16 Dec 2014 22:01:56 -0800 Subject: [PATCH 53/86] update default output to reflect current packer release --- .../source/intro/getting-started/setup.html.markdown | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/website/source/intro/getting-started/setup.html.markdown b/website/source/intro/getting-started/setup.html.markdown index 60eed6df8..b60158815 100644 --- a/website/source/intro/getting-started/setup.html.markdown +++ b/website/source/intro/getting-started/setup.html.markdown @@ -46,10 +46,12 @@ $ packer usage: packer [--version] [--help] [] Available commands are: - build build image(s) from template - fix fixes templates from old versions of packer - inspect see components of a template - validate check that a template is valid + build build image(s) from template + fix fixes templates from old versions of packer + inspect see components of a template + push push template files to a Packer build service + validate check that a template is valid + version Prints the Packer version ``` If you get an error that `packer` could not be found, then your PATH From 59c54a2a7a8e8d37000dd86379a553db621f9332 Mon Sep 17 00:00:00 2001 From: Matt Fellows Date: Fri, 19 Dec 2014 13:32:06 +1100 Subject: [PATCH 54/86] Updated Amazon documentation to reflect the precedence rules putting ~/.aws/credentials files ahead of env vars. Ref #1612 --- website/source/docs/builders/amazon-chroot.html.markdown | 8 ++++---- website/source/docs/builders/amazon-ebs.html.markdown | 8 ++++---- .../source/docs/builders/amazon-instance.html.markdown | 8 ++++---- website/source/docs/builders/amazon.html.markdown | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/website/source/docs/builders/amazon-chroot.html.markdown b/website/source/docs/builders/amazon-chroot.html.markdown index 41235496e..80dc3ab8c 100644 --- a/website/source/docs/builders/amazon-chroot.html.markdown +++ b/website/source/docs/builders/amazon-chroot.html.markdown @@ -54,8 +54,8 @@ each category, the available configuration keys are alphabetized. ### Required: * `access_key` (string) - The access key used to communicate with AWS. - If not specified, Packer will use the environment variables - `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` (in that order), if set. + If not specified, Packer will use the key from any [credentials](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) file + or fall back to environment variables `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` (in that order), if set. If the environmental variables aren't set and Packer is running on an EC2 instance, Packer will check the instance metadata for IAM role keys. @@ -66,8 +66,8 @@ each category, the available configuration keys are alphabetized. [configuration templates](/docs/templates/configuration-templates.html) for more info) * `secret_key` (string) - The secret key used to communicate with AWS. - If not specified, Packer will use the environment variables - `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` (in that order), if set. + If not specified, Packer will use the secret from any [credentials](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) file + or fall back to environment variables `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` (in that order), if set. If the environmental variables aren't set and Packer is running on an EC2 instance, Packer will check the instance metadata for IAM role keys. diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index 4700ade55..9383592fc 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -34,8 +34,8 @@ each category, the available configuration keys are alphabetized. ### Required: * `access_key` (string) - The access key used to communicate with AWS. - If not specified, Packer will use the environment variables - `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` (in that order), if set. + If not specified, Packer will use the key from any [credentials](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) file + or fall back to environment variables `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` (in that order), if set. * `ami_name` (string) - The name of the resulting AMI that will appear when managing AMIs in the AWS console or via APIs. This must be unique. @@ -49,8 +49,8 @@ each category, the available configuration keys are alphabetized. to launch the EC2 instance to create the AMI. * `secret_key` (string) - The secret key used to communicate with AWS. - If not specified, Packer will use the environment variables - `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` (in that order), if set. + If not specified, Packer will use the secret from any [credentials](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) file + or fall back to environment variables `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` (in that order), if set. * `source_ami` (string) - The initial AMI used as a base for the newly created machine. diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown index 6defb57b3..96110aacd 100644 --- a/website/source/docs/builders/amazon-instance.html.markdown +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -39,8 +39,8 @@ each category, the available configuration keys are alphabetized. ### Required: * `access_key` (string) - The access key used to communicate with AWS. - If not specified, Packer will use the environment variables - `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` (in that order), if set. + If not specified, Packer will use the key from any [credentials](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) file + or fall back to environment variables `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` (in that order), if set. * `account_id` (string) - Your AWS account ID. This is required for bundling the AMI. This is _not the same_ as the access key. You can find your @@ -61,8 +61,8 @@ each category, the available configuration keys are alphabetized. This bucket will be created if it doesn't exist. * `secret_key` (string) - The secret key used to communicate with AWS. - If not specified, Packer will use the environment variables - `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` (in that order), if set. + If not specified, Packer will use the secret from any [credentials](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) file + or fall back to environment variables `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` (in that order), if set. * `source_ami` (string) - The initial AMI used as a base for the newly created machine. diff --git a/website/source/docs/builders/amazon.html.markdown b/website/source/docs/builders/amazon.html.markdown index 49d6e202e..ad336ad1c 100644 --- a/website/source/docs/builders/amazon.html.markdown +++ b/website/source/docs/builders/amazon.html.markdown @@ -33,7 +33,7 @@ much easier to use and Amazon generally recommends EBS-backed images nowadays. ## Using an IAM Instance Profile -If AWS keys are not specified in the template or through environment variables +If AWS keys are not specified in the template, a [credentials](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) file or through environment variables Packer will use credentials provided by the instance's IAM profile, if it has one. The following policy document provides the minimal set permissions necessary for Packer to work: From b8626d8340bf7314bb4618e381a631604d32d4cb Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Sun, 21 Dec 2014 10:12:34 -0800 Subject: [PATCH 55/86] Alphabetized options in qemu.html.markdown --- .../source/docs/builders/qemu.html.markdown | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/website/source/docs/builders/qemu.html.markdown b/website/source/docs/builders/qemu.html.markdown index 76fe9e924..34e86f20d 100644 --- a/website/source/docs/builders/qemu.html.markdown +++ b/website/source/docs/builders/qemu.html.markdown @@ -111,17 +111,22 @@ each category, the available options are alphabetized and described. five seconds and one minute 30 seconds, respectively. If this isn't specified, the default is 10 seconds. -* `disk_size` (integer) - The size, in megabytes, of the hard disk to create - for the VM. By default, this is 40000 (about 40 GB). +* `disk_cache` (string) - The cache mode to use for disk. Allowed values + values include any of "writethrough", "writeback", "none", "unsafe" or + "directsync". + +* `disk_image` (boolean) - Packer defaults to building from an ISO file, + this parameter controls whether the ISO URL supplied is actually a bootable + QEMU image. When this value is set to true, the machine will clone the + source, resize it according to `disk_size` and boot the image. * `disk_interface` (string) - The interface to use for the disk. Allowed values include any of "ide," "scsi" or "virtio." Note also that any boot commands or kickstart type scripts must have proper adjustments for resulting device names. The Qemu builder uses "virtio" by default. -* `disk_cache` (string) - The cache mode to use for disk. Allowed values - values include any of "writethrough", "writeback", "none", "unsafe" or - "directsync". +* `disk_size` (integer) - The size, in megabytes, of the hard disk to create + for the VM. By default, this is 40000 (about 40 GB). * `floppy_files` (array of strings) - A list of files to place onto a floppy disk that is attached when the VM is booted. This is most useful @@ -260,11 +265,6 @@ qemu-system-x86 command. The arguments are all printed for review. Packer will choose a randomly available port in this range to use as the host port. -* `disk_image` (boolean) - Packer defaults to building from an ISO file, - this parameter controls whether the ISO URL supplied is actually a bootable - QEMU image. When this value is set to true, the machine will clone the - source, resize it according to `disk_size` and boot the image. - ## Boot Command The `boot_command` configuration is very important: it specifies the keys From 10dee1bd686e7443f7c4aca90cfcbfad970bf0f9 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Sun, 21 Dec 2014 11:22:24 -0800 Subject: [PATCH 56/86] Temporarily fix build by forking golang.org/x/oauth2 @ b3f9a68 --- builder/googlecompute/driver_gce.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/builder/googlecompute/driver_gce.go b/builder/googlecompute/driver_gce.go index 674019f52..13d765ce1 100644 --- a/builder/googlecompute/driver_gce.go +++ b/builder/googlecompute/driver_gce.go @@ -8,8 +8,12 @@ import ( "code.google.com/p/google-api-go-client/compute/v1" "github.com/mitchellh/packer/packer" - "golang.org/x/oauth2" - "golang.org/x/oauth2/google" + + // oauth2 "github.com/rasa/oauth2-fork-b3f9a68" + "github.com/rasa/oauth2-fork-b3f9a68" + + // oauth2 "github.com/rasa/oauth2-fork-b3f9a68/google" + "github.com/rasa/oauth2-fork-b3f9a68/google" ) // driverGCE is a Driver implementation that actually talks to GCE. From f52283d78e1ef9d49deba57e565d2e23d9a19888 Mon Sep 17 00:00:00 2001 From: Godefroid Chapelle Date: Mon, 22 Dec 2014 12:58:41 +0100 Subject: [PATCH 57/86] all calls to cp should be escaped --- builder/docker/communicator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/docker/communicator.go b/builder/docker/communicator.go index 4c5b5043e..bad2c1ff6 100644 --- a/builder/docker/communicator.go +++ b/builder/docker/communicator.go @@ -166,7 +166,7 @@ func (c *Communicator) UploadDir(dst string, src string, exclude []string) error // Make the directory, then copy into it cmd := &packer.RemoteCmd{ - Command: fmt.Sprintf("set -e; mkdir -p %s; cp -R %s/* %s", + Command: fmt.Sprintf("set -e; mkdir -p %s; command cp -R %s/* %s", containerDst, containerSrc, containerDst), } if err := c.Start(cmd); err != nil { From eb725844156c6b306969e69faf2fd69706e267ff Mon Sep 17 00:00:00 2001 From: Victor Trac Date: Mon, 22 Dec 2014 14:08:32 -0600 Subject: [PATCH 58/86] fix passing bootstrap_args to bootstrap script --- provisioner/salt-masterless/provisioner.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/provisioner/salt-masterless/provisioner.go b/provisioner/salt-masterless/provisioner.go index 5223485cb..f7751d242 100644 --- a/provisioner/salt-masterless/provisioner.go +++ b/provisioner/salt-masterless/provisioner.go @@ -114,7 +114,14 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { ui.Say("Provisioning with Salt...") if !p.config.SkipBootstrap { cmd := &packer.RemoteCmd{ - Command: fmt.Sprintf("wget -O - https://bootstrap.saltstack.com | sudo sh -s %s", p.config.BootstrapArgs), + Command: fmt.Sprintf("curl -L https://bootstrap.saltstack.com -o /tmp/install_salt.sh"), + } + ui.Message(fmt.Sprintf("Downloading saltstack bootstrap to /tmp/install_salt.sh")) + if err = cmd.StartWithUi(comm, ui); err != nil { + return fmt.Errorf("Unable to download Salt: %d", err) + } + cmd = &packer.RemoteCmd{ + Command: fmt.Sprintf("sudo sh /tmp/install_salt.sh %s", p.config.BootstrapArgs), } ui.Message(fmt.Sprintf("Installing Salt with command %s", cmd)) if err = cmd.StartWithUi(comm, ui); err != nil { From 07ed577ce0579ed23267195fcb556e911e964ad9 Mon Sep 17 00:00:00 2001 From: Malcolm Locke Date: Wed, 24 Dec 2014 13:37:41 +1300 Subject: [PATCH 59/86] Fix link to shell provisioner --- .../source/docs/provisioners/puppet-masterless.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/provisioners/puppet-masterless.html.markdown b/website/source/docs/provisioners/puppet-masterless.html.markdown index b3574cac6..953dca6ea 100644 --- a/website/source/docs/provisioners/puppet-masterless.html.markdown +++ b/website/source/docs/provisioners/puppet-masterless.html.markdown @@ -19,7 +19,7 @@ master. -> **Note:** Puppet will _not_ be installed automatically by this provisioner. This provisioner expects that Puppet is already installed on the machine. It is common practice to use the -[shell provisioner[(/docs/provisioners/shell.html) before the +[shell provisioner](/docs/provisioners/shell.html) before the Puppet provisioner to do this. ## Basic Example From cf23e3440403753e5620ca1262e34ab4d39dae43 Mon Sep 17 00:00:00 2001 From: Joel Handwell Date: Tue, 23 Dec 2014 20:59:48 -0500 Subject: [PATCH 60/86] Added Chocolatey as alternative installation --- website/source/docs/installation.html.markdown | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/website/source/docs/installation.html.markdown b/website/source/docs/installation.html.markdown index c0aee9528..e98a7acae 100644 --- a/website/source/docs/installation.html.markdown +++ b/website/source/docs/installation.html.markdown @@ -73,3 +73,13 @@ it may not be the latest available version. $ brew tap homebrew/binary $ brew install packer ``` + +### Chocolatey + +If you're using Windows and [Chocolatey](http://chocolatey.org), you can install Packer from +Windows command line (cmd). Remember that this is updated by a 3rd party, so +it may not be the latest available version. + +```text +$ choco install packer +``` From 297eb3777b2205ff6e6c1feee00c07a6a7511d66 Mon Sep 17 00:00:00 2001 From: Joel Handwell Date: Fri, 26 Dec 2014 00:42:41 -0500 Subject: [PATCH 61/86] Fix typo: One attached -> Once attached --- website/source/docs/builders/amazon-chroot.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/builders/amazon-chroot.html.markdown b/website/source/docs/builders/amazon-chroot.html.markdown index 41235496e..11eea6eb3 100644 --- a/website/source/docs/builders/amazon-chroot.html.markdown +++ b/website/source/docs/builders/amazon-chroot.html.markdown @@ -30,7 +30,7 @@ in your account, it is up to you to use, delete, etc. the AMI. ## How Does it Work? This builder works by creating a new EBS volume from an existing source AMI -and attaching it into an already-running EC2 instance. One attached, a +and attaching it into an already-running EC2 instance. Once attached, a [chroot](http://en.wikipedia.org/wiki/Chroot) is used to provision the system within that volume. After provisioning, the volume is detached, snapshotted, and an AMI is made. From 277632ad99f1bae60cab3d8b4f204d5ef6f782cc Mon Sep 17 00:00:00 2001 From: Johannes Graf Date: Wed, 24 Dec 2014 02:19:42 +0100 Subject: [PATCH 62/86] Removed version.number / vagrant-cloud post-processor A few weeks ago ... the version.number changed from an integer to a semver based string. I guess version.number and version.version are equals now. So version.version can be used. This commit should fix #1735 --- .../vagrant-cloud/step_create_provider.go | 4 ++-- .../vagrant-cloud/step_create_version.go | 18 +++--------------- .../vagrant-cloud/step_prepare_upload.go | 2 +- .../vagrant-cloud/step_release_version.go | 2 +- .../vagrant-cloud/step_verify_upload.go | 2 +- 5 files changed, 8 insertions(+), 20 deletions(-) diff --git a/post-processor/vagrant-cloud/step_create_provider.go b/post-processor/vagrant-cloud/step_create_provider.go index c2d6772bd..6e19f3f3e 100644 --- a/post-processor/vagrant-cloud/step_create_provider.go +++ b/post-processor/vagrant-cloud/step_create_provider.go @@ -25,7 +25,7 @@ func (s *stepCreateProvider) Run(state multistep.StateBag) multistep.StepAction providerName := state.Get("providerName").(string) downloadUrl := state.Get("boxDownloadUrl").(string) - path := fmt.Sprintf("box/%s/version/%v/providers", box.Tag, version.Number) + path := fmt.Sprintf("box/%s/version/%v/providers", box.Tag, version.Version) provider := &Provider{Name: providerName} @@ -86,7 +86,7 @@ func (s *stepCreateProvider) Cleanup(state multistep.StateBag) { ui.Say("Cleaning up provider") ui.Message(fmt.Sprintf("Deleting provider: %s", s.name)) - path := fmt.Sprintf("box/%s/version/%v/provider/%s", box.Tag, version.Number, s.name) + path := fmt.Sprintf("box/%s/version/%v/provider/%s", box.Tag, version.Version, s.name) // No need for resp from the cleanup DELETE _, err := client.Delete(path) diff --git a/post-processor/vagrant-cloud/step_create_version.go b/post-processor/vagrant-cloud/step_create_version.go index 8108577cc..738614152 100644 --- a/post-processor/vagrant-cloud/step_create_version.go +++ b/post-processor/vagrant-cloud/step_create_version.go @@ -9,11 +9,9 @@ import ( type Version struct { Version string `json:"version"` Description string `json:"description,omitempty"` - Number uint `json:"number,omitempty"` } type stepCreateVersion struct { - number uint // number of the version, if needed in cleanup } func (s *stepCreateVersion) Run(state multistep.StateBag) multistep.StepAction { @@ -52,9 +50,6 @@ func (s *stepCreateVersion) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - // Save the number for cleanup - s.number = version.Number - state.Put("version", version) return multistep.ActionContinue @@ -63,15 +58,8 @@ func (s *stepCreateVersion) Run(state multistep.StateBag) multistep.StepAction { func (s *stepCreateVersion) Cleanup(state multistep.StateBag) { client := state.Get("client").(*VagrantCloudClient) ui := state.Get("ui").(packer.Ui) - config := state.Get("config").(Config) box := state.Get("box").(*Box) - - // If we didn't save the version number, it likely doesn't exist or - // already existed - if s.number == 0 { - ui.Message("Version was not created or previously existed, not deleting") - return - } + version := state.Get("version").(*Version) _, cancelled := state.GetOk(multistep.StateCancelled) _, halted := state.GetOk(multistep.StateHalted) @@ -82,10 +70,10 @@ func (s *stepCreateVersion) Cleanup(state multistep.StateBag) { return } - path := fmt.Sprintf("box/%s/version/%v", box.Tag, s.number) + path := fmt.Sprintf("box/%s/version/%v", box.Tag, version.Version) ui.Say("Cleaning up version") - ui.Message(fmt.Sprintf("Deleting version: %s", config.Version)) + ui.Message(fmt.Sprintf("Deleting version: %s", version.Version)) // No need for resp from the cleanup DELETE _, err := client.Delete(path) diff --git a/post-processor/vagrant-cloud/step_prepare_upload.go b/post-processor/vagrant-cloud/step_prepare_upload.go index 5c82d02c3..d28a9fd2b 100644 --- a/post-processor/vagrant-cloud/step_prepare_upload.go +++ b/post-processor/vagrant-cloud/step_prepare_upload.go @@ -22,7 +22,7 @@ func (s *stepPrepareUpload) Run(state multistep.StateBag) multistep.StepAction { provider := state.Get("provider").(*Provider) artifactFilePath := state.Get("artifactFilePath").(string) - path := fmt.Sprintf("box/%s/version/%v/provider/%s/upload", box.Tag, version.Number, provider.Name) + path := fmt.Sprintf("box/%s/version/%v/provider/%s/upload", box.Tag, version.Version, provider.Name) upload := &Upload{} ui.Say(fmt.Sprintf("Preparing upload of box: %s", artifactFilePath)) diff --git a/post-processor/vagrant-cloud/step_release_version.go b/post-processor/vagrant-cloud/step_release_version.go index dab417d00..6301d5f19 100644 --- a/post-processor/vagrant-cloud/step_release_version.go +++ b/post-processor/vagrant-cloud/step_release_version.go @@ -24,7 +24,7 @@ func (s *stepReleaseVersion) Run(state multistep.StateBag) multistep.StepAction return multistep.ActionContinue } - path := fmt.Sprintf("box/%s/version/%v/release", box.Tag, version.Number) + path := fmt.Sprintf("box/%s/version/%v/release", box.Tag, version.Version) resp, err := client.Put(path) diff --git a/post-processor/vagrant-cloud/step_verify_upload.go b/post-processor/vagrant-cloud/step_verify_upload.go index 120bd647d..fbed25c6c 100644 --- a/post-processor/vagrant-cloud/step_verify_upload.go +++ b/post-processor/vagrant-cloud/step_verify_upload.go @@ -19,7 +19,7 @@ func (s *stepVerifyUpload) Run(state multistep.StateBag) multistep.StepAction { upload := state.Get("upload").(*Upload) provider := state.Get("provider").(*Provider) - path := fmt.Sprintf("box/%s/version/%v/provider/%s", box.Tag, version.Number, provider.Name) + path := fmt.Sprintf("box/%s/version/%v/provider/%s", box.Tag, version.Version, provider.Name) providerCheck := &Provider{} From d424ed1e37c61262f36050b43ba07bb59afe0b5a Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Thu, 11 Dec 2014 22:09:04 +0100 Subject: [PATCH 63/86] builder/parallels: Use most recent DHCP lease when determening IP. Fixes #1746. Problem establishing SSH connection with Fedora 21 and Parallels --- builder/parallels/common/driver.go | 7 +++- builder/parallels/common/driver_9.go | 43 ++++++++++++------- builder/parallels/common/driver_9_test.go | 51 +++++++++++++++++++++++ 3 files changed, 85 insertions(+), 16 deletions(-) diff --git a/builder/parallels/common/driver.go b/builder/parallels/common/driver.go index 771395d8c..bdb73719c 100644 --- a/builder/parallels/common/driver.go +++ b/builder/parallels/common/driver.go @@ -55,6 +55,7 @@ func NewDriver() (Driver, error) { var drivers map[string]Driver var prlctlPath string var supportedVersions []string + dhcp_lease_file := "/Library/Preferences/Parallels/parallels_dhcp_leases" if runtime.GOOS != "darwin" { return nil, fmt.Errorf( @@ -74,11 +75,13 @@ func NewDriver() (Driver, error) { drivers = map[string]Driver{ "10": &Parallels10Driver{ Parallels9Driver: Parallels9Driver{ - PrlctlPath: prlctlPath, + PrlctlPath: prlctlPath, + dhcp_lease_file: dhcp_lease_file, }, }, "9": &Parallels9Driver{ - PrlctlPath: prlctlPath, + PrlctlPath: prlctlPath, + dhcp_lease_file: dhcp_lease_file, }, } diff --git a/builder/parallels/common/driver_9.go b/builder/parallels/common/driver_9.go index 7661a30c7..98d36cc24 100644 --- a/builder/parallels/common/driver_9.go +++ b/builder/parallels/common/driver_9.go @@ -9,6 +9,7 @@ import ( "os/exec" "path/filepath" "regexp" + "strconv" "strings" "time" @@ -18,6 +19,8 @@ import ( type Parallels9Driver struct { // This is the path to the "prlctl" application. PrlctlPath string + // The path to the parallels_dhcp_leases file + dhcp_lease_file string } func (d *Parallels9Driver) Import(name, srcPath, dstDir string, reassignMac bool) error { @@ -276,31 +279,43 @@ func (d *Parallels9Driver) Mac(vmName string) (string, error) { } // Finds the IP address of a VM connected that uses DHCP by its MAC address +// +// Parses the file /Library/Preferences/Parallels/parallels_dhcp_leases +// file contain a list of DHCP leases given by Parallels Desktop +// Example line: +// 10.211.55.181="1418921112,1800,001c42f593fb,ff42f593fb000100011c25b9ff001c42f593fb" +// IP Address ="Lease expiry, Lease time, MAC, MAC or DUID" func (d *Parallels9Driver) IpAddress(mac string) (string, error) { - var stdout bytes.Buffer - dhcp_lease_file := "/Library/Preferences/Parallels/parallels_dhcp_leases" if len(mac) != 12 { return "", fmt.Errorf("Not a valid MAC address: %s. It should be exactly 12 digits.", mac) } - cmd := exec.Command("grep", "-i", mac, dhcp_lease_file) - cmd.Stdout = &stdout - if err := cmd.Run(); err != nil { + leases, err := ioutil.ReadFile(d.dhcp_lease_file) + if err != nil { return "", err } - stdoutString := strings.TrimSpace(stdout.String()) - re := regexp.MustCompile("(.*)=.*") - ipMatch := re.FindAllStringSubmatch(stdoutString, 1) - - if len(ipMatch) != 1 { - return "", fmt.Errorf("IP lease not found for MAC address %s in: %s\n", mac, dhcp_lease_file) + re := regexp.MustCompile("(.*)=\"(.*),(.*)," + strings.ToLower(mac) + ",.*\"") + mostRecentIp := "" + mostRecentLease := uint64(0) + for _, l := range re.FindAllStringSubmatch(string(leases), -1) { + ip := l[1] + expiry, _ := strconv.ParseUint(l[2], 10, 64) + leaseTime, _ := strconv.ParseUint(l[3], 10, 32) + log.Printf("Found lease: %s for MAC: %s, expiring at %d, leased for %d s.\n", ip, mac, expiry, leaseTime) + if mostRecentLease <= expiry-leaseTime { + mostRecentIp = ip + mostRecentLease = expiry - leaseTime + } } - ip := ipMatch[0][1] - log.Printf("Found IP lease: %s for MAC address %s\n", ip, mac) - return ip, nil + if len(mostRecentIp) == 0 { + return "", fmt.Errorf("IP lease not found for MAC address %s in: %s\n", mac, d.dhcp_lease_file) + } + + log.Printf("Found IP lease: %s for MAC address %s\n", mostRecentIp, mac) + return mostRecentIp, nil } func (d *Parallels9Driver) ToolsIsoPath(k string) (string, error) { diff --git a/builder/parallels/common/driver_9_test.go b/builder/parallels/common/driver_9_test.go index a7c61b9d2..02fb4faf8 100644 --- a/builder/parallels/common/driver_9_test.go +++ b/builder/parallels/common/driver_9_test.go @@ -1,9 +1,60 @@ package common import ( + "io/ioutil" + "os" "testing" ) func TestParallels9Driver_impl(t *testing.T) { var _ Driver = new(Parallels9Driver) } + +func TestIpAddress(t *testing.T) { + tf, err := ioutil.TempFile("", "packer") + if err != nil { + t.Fatalf("err: %s", err) + } + defer os.Remove(tf.Name()) + + d := Parallels9Driver{ + dhcp_lease_file: tf.Name(), + } + + // No lease should be found in an empty file + ip, err := d.IpAddress("123456789012") + if err == nil { + t.Fatalf("Found IP: \"%v\". No IP should be found!\n", ip) + } + + // The most recent lease, 10.211.55.126 should be found + c := []byte(` +[vnic0] +10.211.55.125="1418288000,1800,001c4235240c,ff4235240c000100011c1c10e7001c4235240c" +10.211.55.126="1418288969,1800,001c4235240c,ff4235240c000100011c1c11ad001c4235240c" +10.211.55.254="1411712008,1800,001c42a51419,01001c42a51419" +`) + ioutil.WriteFile(tf.Name(), c, 0666) + ip, err = d.IpAddress("001C4235240c") + if err != nil { + t.Fatalf("Error: %v\n", err) + } + if ip != "10.211.55.126" { + t.Fatalf("Should have found 10.211.55.126, not %s!\n", ip) + } + + // The most recent lease, 10.211.55.124 should be found + c = []byte(`[vnic0] +10.211.55.124="1418288969,1800,001c4235240c,ff4235240c000100011c1c11ad001c4235240c" +10.211.55.125="1418288000,1800,001c4235240c,ff4235240c000100011c1c10e7001c4235240c" +10.211.55.254="1411712008,1800,001c42a51419,01001c42a51419" +`) + ioutil.WriteFile(tf.Name(), c, 0666) + ip, err = d.IpAddress("001c4235240c") + if err != nil { + t.Fatalf("Error: %v\n", err) + } + if ip != "10.211.55.124" { + t.Fatalf("Should have found 10.211.55.124, not %s!\n", ip) + } +} From 92ca37679521200512310aec96d2e17b34e1322f Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Mon, 29 Dec 2014 14:04:03 -0800 Subject: [PATCH 64/86] Fixed failing on response [GH-1735] --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 492e96ea8..c964bf718 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## 0.8.0 (unreleased) + +FEATURES: + +IMPROVEMENTS: + +BUG FIXES: + + * postprocessor/vagrant-cloud: Fixed failing on response + ## 0.7.5 (December 9, 2014) FEATURES: From acb166aad78ac736e0113a6deeaea5c144c592f5 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Mon, 29 Dec 2014 14:07:59 -0800 Subject: [PATCH 65/86] builder/docker: Fixed hang on prompt while copying script, fixes #1656 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c964bf718..65ed7e795 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ IMPROVEMENTS: BUG FIXES: * postprocessor/vagrant-cloud: Fixed failing on response + * builder/docker: Fixed hang on prompt while copying script ## 0.7.5 (December 9, 2014) From a65c67fab7aa0e8a4677d0c1a223a54b497aae57 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Mon, 29 Dec 2014 14:08:20 -0800 Subject: [PATCH 66/86] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65ed7e795..eb8d980c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ IMPROVEMENTS: BUG FIXES: - * postprocessor/vagrant-cloud: Fixed failing on response * builder/docker: Fixed hang on prompt while copying script + * postprocessor/vagrant-cloud: Fixed failing on response ## 0.7.5 (December 9, 2014) From 2878e68a7de589fd1ac9581b4b5b1b6c5d79f7d0 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Mon, 29 Dec 2014 19:41:01 -0800 Subject: [PATCH 67/86] provisioner/puppet-masterless: Allow manifest_file to be a directory [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb8d980c2..3ebb5c6c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ BUG FIXES: * builder/docker: Fixed hang on prompt while copying script * postprocessor/vagrant-cloud: Fixed failing on response + * provisioner/puppet-masterless: Allow manifest_file to be a directory ## 0.7.5 (December 9, 2014) From 034ed36d0e665c068e71606c0f0de0691244a8bc Mon Sep 17 00:00:00 2001 From: Erik Simmler Date: Tue, 30 Dec 2014 10:33:59 -0500 Subject: [PATCH 68/86] Ask salt to return a proper exit code so we can fail the builder appropriately. --- provisioner/salt-masterless/provisioner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioner/salt-masterless/provisioner.go b/provisioner/salt-masterless/provisioner.go index f7751d242..07710cdd5 100644 --- a/provisioner/salt-masterless/provisioner.go +++ b/provisioner/salt-masterless/provisioner.go @@ -181,7 +181,7 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { } ui.Message("Running highstate") - cmd := &packer.RemoteCmd{Command: "sudo salt-call --local state.highstate -l info"} + cmd := &packer.RemoteCmd{Command: "sudo salt-call --local state.highstate -l info --retcode-passthrough"} if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 { if err == nil { err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus) From c0a56b6a2b3c2e3285f4306ca4db2e507b25d537 Mon Sep 17 00:00:00 2001 From: Nick Dainty Date: Tue, 30 Dec 2014 19:55:16 +0000 Subject: [PATCH 69/86] Fix amazon instance type on build an image page This page referred to using a "t1.micro" instance but the template actually uses a "t2.micro". --- website/source/intro/getting-started/build-image.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/intro/getting-started/build-image.html.markdown b/website/source/intro/getting-started/build-image.html.markdown index ec6861d2e..9e4a4dedd 100644 --- a/website/source/intro/getting-started/build-image.html.markdown +++ b/website/source/intro/getting-started/build-image.html.markdown @@ -16,7 +16,7 @@ with Redis pre-installed. This is just an example. Packer can create images for [many platforms](/intro/platforms.html) with anything pre-installed. If you don't have an AWS account, [create one now](http://aws.amazon.com/free/). -For the example, we'll use a "t1.micro" instance to build our image, which +For the example, we'll use a "t2.micro" instance to build our image, which qualifies under the AWS [free-tier](http://aws.amazon.com/free/), meaning it will be free. If you already have an AWS account, you may be charged some amount of money, but it shouldn't be more than a few cents. From 082f9244ed37c485a7c111f088b84dc3415cfb34 Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Wed, 31 Dec 2014 07:32:38 -0800 Subject: [PATCH 70/86] provisioner/salt-masterless: Add `--retcode-passthrough` to salt-call [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ebb5c6c2..4371fb1b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ BUG FIXES: * builder/docker: Fixed hang on prompt while copying script * postprocessor/vagrant-cloud: Fixed failing on response * provisioner/puppet-masterless: Allow manifest_file to be a directory + * provisioner/salt-masterless: Add `--retcode-passthrough` to salt-call ## 0.7.5 (December 9, 2014) From 6b14b6282a11451122eff987581373e91e7bedec Mon Sep 17 00:00:00 2001 From: Timo Sugliani Date: Sat, 3 Jan 2015 14:34:37 +0100 Subject: [PATCH 71/86] Update plugins.html.markdown I think this is the required PATH for windows. --- website/source/docs/extend/plugins.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/extend/plugins.html.markdown b/website/source/docs/extend/plugins.html.markdown index 12db22150..c257fb702 100644 --- a/website/source/docs/extend/plugins.html.markdown +++ b/website/source/docs/extend/plugins.html.markdown @@ -50,7 +50,7 @@ found later, it will take precedence over one found earlier. 1. The directory where `packer` is, or the executable directory. -2. `~/.packer.d/plugins` on Unix systems or `%APPDATA%/packer.d` on +2. `~/.packer.d/plugins` on Unix systems or `%APPDATA%/packer.d/plugins` on Windows. 3. The current working directory. From 4b0d86137302664853de718805d191201e15b5db Mon Sep 17 00:00:00 2001 From: Jack Pearkes Date: Sun, 4 Jan 2015 13:43:54 -0500 Subject: [PATCH 72/86] docs/vmware-iso: fix boot command encoding --- website/source/docs/builders/vmware-iso.html.markdown | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/source/docs/builders/vmware-iso.html.markdown b/website/source/docs/builders/vmware-iso.html.markdown index d909aac0f..b2768d727 100644 --- a/website/source/docs/builders/vmware-iso.html.markdown +++ b/website/source/docs/builders/vmware-iso.html.markdown @@ -313,7 +313,7 @@ an Ubuntu 12.04 installer: ```javascript [ - "<esc><esc><enter><wait>", + "", "/install/vmlinuz noapic ", "preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ", "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ", @@ -321,7 +321,7 @@ an Ubuntu 12.04 installer: "fb=false debconf/frontend=noninteractive ", "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ", "keyboard-configuration/variant=USA console-setup/ask_detect=false ", - "initrd=/install/initrd.gz -- <enter>" + "initrd=/install/initrd.gz -- " ] ``` From 555b89567f0a60624eea53262f91c08aba382aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20R=C3=B6der?= Date: Tue, 13 Jan 2015 22:43:24 +0100 Subject: [PATCH 73/86] s/VirtualBox/VMware in comment --- builder/vmware/vmx/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/vmware/vmx/builder.go b/builder/vmware/vmx/builder.go index 100e1fe36..4597e647b 100644 --- a/builder/vmware/vmx/builder.go +++ b/builder/vmware/vmx/builder.go @@ -12,7 +12,7 @@ import ( "github.com/mitchellh/packer/packer" ) -// Builder implements packer.Builder and builds the actual VirtualBox +// Builder implements packer.Builder and builds the actual VMware // images. type Builder struct { config *Config From 84d6d856f69162eb55ec1fc399bca1383e941e05 Mon Sep 17 00:00:00 2001 From: Danny Lockard Date: Thu, 15 Jan 2015 19:53:01 -0600 Subject: [PATCH 74/86] Add the ability to create a SCSI Device as the main HD controller for virtualbox. --- builder/virtualbox/common/driver.go | 3 +++ builder/virtualbox/common/driver_4_2.go | 12 ++++++++++++ builder/virtualbox/common/driver_mock.go | 10 ++++++++++ builder/virtualbox/iso/builder.go | 4 ++-- builder/virtualbox/iso/step_create_disk.go | 13 +++++++++++++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/builder/virtualbox/common/driver.go b/builder/virtualbox/common/driver.go index a55b20199..9fb82f7da 100644 --- a/builder/virtualbox/common/driver.go +++ b/builder/virtualbox/common/driver.go @@ -18,6 +18,9 @@ import ( type Driver interface { // Create a SATA controller. CreateSATAController(vm string, controller string) error + + // Create a SCSI controller. + CreateSCSIController(vm string, controller string) error // Delete a VM by name Delete(string) error diff --git a/builder/virtualbox/common/driver_4_2.go b/builder/virtualbox/common/driver_4_2.go index 3462933cd..f53a7ac4b 100644 --- a/builder/virtualbox/common/driver_4_2.go +++ b/builder/virtualbox/common/driver_4_2.go @@ -36,6 +36,18 @@ func (d *VBox42Driver) CreateSATAController(vmName string, name string) error { return d.VBoxManage(command...) } +func (d *VBox42Driver) CreateSCSIController(vmName string, name string) error { + + command := []string{ + "storagectl", vmName, + "--name", name, + "--add", "scsi", + "--controller", "LSILogic", + } + + return d.VBoxManage(command...) +} + func (d *VBox42Driver) Delete(name string) error { return d.VBoxManage("unregistervm", name, "--delete") } diff --git a/builder/virtualbox/common/driver_mock.go b/builder/virtualbox/common/driver_mock.go index 1aacf9f8f..b0f5dbe94 100644 --- a/builder/virtualbox/common/driver_mock.go +++ b/builder/virtualbox/common/driver_mock.go @@ -9,6 +9,10 @@ type DriverMock struct { CreateSATAControllerController string CreateSATAControllerErr error + CreateSCSIControllerVM string + CreateSCSIControllerController string + CreateSCSIControllerErr error + DeleteCalled bool DeleteName string DeleteErr error @@ -49,6 +53,12 @@ func (d *DriverMock) CreateSATAController(vm string, controller string) error { return d.CreateSATAControllerErr } +func (d *DriverMock) CreateSCSIController(vm string, controller string) error { + d.CreateSCSIControllerVM = vm + d.CreateSCSIControllerController = vm + return d.CreateSCSIControllerErr +} + func (d *DriverMock) Delete(name string) error { d.DeleteCalled = true d.DeleteName = name diff --git a/builder/virtualbox/iso/builder.go b/builder/virtualbox/iso/builder.go index c4c51c892..826d3ffe9 100644 --- a/builder/virtualbox/iso/builder.go +++ b/builder/virtualbox/iso/builder.go @@ -158,9 +158,9 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { } } - if b.config.HardDriveInterface != "ide" && b.config.HardDriveInterface != "sata" { + if b.config.HardDriveInterface != "ide" && b.config.HardDriveInterface != "sata" && b.config.HardDriveInterface != "scsi" { errs = packer.MultiErrorAppend( - errs, errors.New("hard_drive_interface can only be ide or sata")) + errs, errors.New("hard_drive_interface can only be ide, sata, or scsi")) } if b.config.ISOChecksumType == "" { diff --git a/builder/virtualbox/iso/step_create_disk.go b/builder/virtualbox/iso/step_create_disk.go index 564a2f4d3..626ca9b28 100644 --- a/builder/virtualbox/iso/step_create_disk.go +++ b/builder/virtualbox/iso/step_create_disk.go @@ -63,11 +63,24 @@ func (s *stepCreateDisk) Run(state multistep.StateBag) multistep.StepAction { } } + if config.HardDriveInterface == "scsi" { + if err := driver.CreateSCSIController(vmName, "SCSI Controller"); err != nil { + err := fmt.Errorf("Error creating disk controller: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + } + // Attach the disk to the controller controllerName := "IDE Controller" if config.HardDriveInterface == "sata" { controllerName = "SATA Controller" } + + if config.HardDriveInterface == "scsi" { + controllerName = "SCSI Controller" + } command = []string{ "storageattach", vmName, From fdcf10dc64414cfeb186a41cd47cb76fb05376f1 Mon Sep 17 00:00:00 2001 From: Danny Lockard Date: Thu, 15 Jan 2015 19:56:30 -0600 Subject: [PATCH 75/86] and update the docs to indicate you can use 'scsi' in place of sata or ide --- website/source/docs/builders/virtualbox-iso.html.markdown | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/source/docs/builders/virtualbox-iso.html.markdown b/website/source/docs/builders/virtualbox-iso.html.markdown index f8a45d352..b97045f16 100644 --- a/website/source/docs/builders/virtualbox-iso.html.markdown +++ b/website/source/docs/builders/virtualbox-iso.html.markdown @@ -136,7 +136,8 @@ each category, the available options are alphabetized and described. * `hard_drive_interface` (string) - The type of controller that the primary hard drive is attached to, defaults to "ide". When set to "sata", the - drive is attached to an AHCI SATA controller. + drive is attached to an AHCI SATA controller. When set to "scsi", the drive + is attached to an LsiLogic SCSI controller. * `headless` (boolean) - Packer defaults to building VirtualBox virtual machines by launching a GUI that shows the console of the From 7d1b95c907383a4e67a7e0db0035fe826b5d446f Mon Sep 17 00:00:00 2001 From: Danny Lockard Date: Fri, 16 Jan 2015 10:34:12 -0600 Subject: [PATCH 76/86] Formatting fix :) --- builder/virtualbox/common/driver.go | 6 +++--- builder/virtualbox/common/driver_4_2.go | 2 +- builder/virtualbox/common/driver_mock.go | 8 ++++---- builder/virtualbox/iso/step_create_disk.go | 24 +++++++++++----------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/builder/virtualbox/common/driver.go b/builder/virtualbox/common/driver.go index 9fb82f7da..add59c7f1 100644 --- a/builder/virtualbox/common/driver.go +++ b/builder/virtualbox/common/driver.go @@ -18,9 +18,9 @@ import ( type Driver interface { // Create a SATA controller. CreateSATAController(vm string, controller string) error - - // Create a SCSI controller. - CreateSCSIController(vm string, controller string) error + + // Create a SCSI controller. + CreateSCSIController(vm string, controller string) error // Delete a VM by name Delete(string) error diff --git a/builder/virtualbox/common/driver_4_2.go b/builder/virtualbox/common/driver_4_2.go index f53a7ac4b..c375fba32 100644 --- a/builder/virtualbox/common/driver_4_2.go +++ b/builder/virtualbox/common/driver_4_2.go @@ -42,7 +42,7 @@ func (d *VBox42Driver) CreateSCSIController(vmName string, name string) error { "storagectl", vmName, "--name", name, "--add", "scsi", - "--controller", "LSILogic", + "--controller", "LSILogic", } return d.VBoxManage(command...) diff --git a/builder/virtualbox/common/driver_mock.go b/builder/virtualbox/common/driver_mock.go index b0f5dbe94..da08ad072 100644 --- a/builder/virtualbox/common/driver_mock.go +++ b/builder/virtualbox/common/driver_mock.go @@ -9,7 +9,7 @@ type DriverMock struct { CreateSATAControllerController string CreateSATAControllerErr error - CreateSCSIControllerVM string + CreateSCSIControllerVM string CreateSCSIControllerController string CreateSCSIControllerErr error @@ -54,9 +54,9 @@ func (d *DriverMock) CreateSATAController(vm string, controller string) error { } func (d *DriverMock) CreateSCSIController(vm string, controller string) error { - d.CreateSCSIControllerVM = vm - d.CreateSCSIControllerController = vm - return d.CreateSCSIControllerErr + d.CreateSCSIControllerVM = vm + d.CreateSCSIControllerController = vm + return d.CreateSCSIControllerErr } func (d *DriverMock) Delete(name string) error { diff --git a/builder/virtualbox/iso/step_create_disk.go b/builder/virtualbox/iso/step_create_disk.go index 626ca9b28..82b849dbe 100644 --- a/builder/virtualbox/iso/step_create_disk.go +++ b/builder/virtualbox/iso/step_create_disk.go @@ -63,24 +63,24 @@ func (s *stepCreateDisk) Run(state multistep.StateBag) multistep.StepAction { } } - if config.HardDriveInterface == "scsi" { - if err := driver.CreateSCSIController(vmName, "SCSI Controller"); err != nil { - err := fmt.Errorf("Error creating disk controller: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - } + if config.HardDriveInterface == "scsi" { + if err := driver.CreateSCSIController(vmName, "SCSI Controller"); err != nil { + err := fmt.Errorf("Error creating disk controller: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + } // Attach the disk to the controller controllerName := "IDE Controller" if config.HardDriveInterface == "sata" { controllerName = "SATA Controller" } - - if config.HardDriveInterface == "scsi" { - controllerName = "SCSI Controller" - } + + if config.HardDriveInterface == "scsi" { + controllerName = "SCSI Controller" + } command = []string{ "storageattach", vmName, From 57ce0e54c955d17366f9f16aca4abf9159b4947f Mon Sep 17 00:00:00 2001 From: Ross Smith II Date: Sat, 17 Jan 2015 09:28:43 -0800 Subject: [PATCH 77/86] builder/virtualbox: Added SCSI support [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4371fb1b8..c03894f7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ IMPROVEMENTS: BUG FIXES: * builder/docker: Fixed hang on prompt while copying script + * builder/virtualbox: Added SCSI support * postprocessor/vagrant-cloud: Fixed failing on response * provisioner/puppet-masterless: Allow manifest_file to be a directory * provisioner/salt-masterless: Add `--retcode-passthrough` to salt-call From e696bb7727bfe7c0f334f5d658f38cc845b970a2 Mon Sep 17 00:00:00 2001 From: Misha Brukman Date: Tue, 20 Jan 2015 10:59:31 -0500 Subject: [PATCH 78/86] provisioner/shell: Fix link with parentheses. --- website/source/docs/provisioners/shell.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/provisioners/shell.html.markdown b/website/source/docs/provisioners/shell.html.markdown index ffed1e442..afee187df 100644 --- a/website/source/docs/provisioners/shell.html.markdown +++ b/website/source/docs/provisioners/shell.html.markdown @@ -150,7 +150,7 @@ on reboot or in your shell script. For example, on Gentoo: * On Ubuntu the /bin/sh shell is [dash](http://en.wikipedia.org/wiki/Debian_Almquist_shell). If your script has -[bash](http://en.wikipedia.org/wiki/Bash_(Unix_shell\)) specific commands in it +[bash](http://en.wikipedia.org/wiki/Bash_(Unix_shell)) specific commands in it then put `#!/bin/bash` at the top of your script. Differences between dash and bash can be found on the [DashAsBinSh](https://wiki.ubuntu.com/DashAsBinSh) Ubuntu wiki page. From 239bdcef29b54b31355aadfcd98c232a09cea1d7 Mon Sep 17 00:00:00 2001 From: Misha Brukman Date: Tue, 20 Jan 2015 11:01:37 -0500 Subject: [PATCH 79/86] provisioner/shell: Added code formatting. --- website/source/docs/provisioners/shell.html.markdown | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/source/docs/provisioners/shell.html.markdown b/website/source/docs/provisioners/shell.html.markdown index afee187df..b1b91de37 100644 --- a/website/source/docs/provisioners/shell.html.markdown +++ b/website/source/docs/provisioners/shell.html.markdown @@ -148,7 +148,7 @@ on reboot or in your shell script. For example, on Gentoo: *My shell script doesn't work correctly on Ubuntu* -* On Ubuntu the /bin/sh shell is +* On Ubuntu the `/bin/sh` shell is [dash](http://en.wikipedia.org/wiki/Debian_Almquist_shell). If your script has [bash](http://en.wikipedia.org/wiki/Bash_(Unix_shell)) specific commands in it then put `#!/bin/bash` at the top of your script. Differences @@ -156,12 +156,12 @@ between dash and bash can be found on the [DashAsBinSh](https://wiki.ubuntu.com/ *My shell works when I login but fails with the shell provisioner* -* See the above tip. More than likely your login shell is using /bin/bash -while the provisioner is using /bin/sh. +* See the above tip. More than likely your login shell is using `/bin/bash` +while the provisioner is using `/bin/sh`. *My installs hang when using `apt-get` or `yum`* -* Make sure you add a "-y" to the command to prevent it from requiring +* Make sure you add a `-y` to the command to prevent it from requiring user input before proceeding. *How do I tell what my shell script is doing?* From 61c95732dd0bd7fa3e7401d220c8f178d6c41e64 Mon Sep 17 00:00:00 2001 From: Misha Brukman Date: Tue, 20 Jan 2015 11:04:38 -0500 Subject: [PATCH 80/86] provisioner/shell: Added punctuation. --- website/source/docs/provisioners/shell.html.markdown | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/source/docs/provisioners/shell.html.markdown b/website/source/docs/provisioners/shell.html.markdown index b1b91de37..41653abf5 100644 --- a/website/source/docs/provisioners/shell.html.markdown +++ b/website/source/docs/provisioners/shell.html.markdown @@ -148,15 +148,15 @@ on reboot or in your shell script. For example, on Gentoo: *My shell script doesn't work correctly on Ubuntu* -* On Ubuntu the `/bin/sh` shell is +* On Ubuntu, the `/bin/sh` shell is [dash](http://en.wikipedia.org/wiki/Debian_Almquist_shell). If your script has -[bash](http://en.wikipedia.org/wiki/Bash_(Unix_shell)) specific commands in it +[bash](http://en.wikipedia.org/wiki/Bash_(Unix_shell))-specific commands in it, then put `#!/bin/bash` at the top of your script. Differences between dash and bash can be found on the [DashAsBinSh](https://wiki.ubuntu.com/DashAsBinSh) Ubuntu wiki page. *My shell works when I login but fails with the shell provisioner* -* See the above tip. More than likely your login shell is using `/bin/bash` +* See the above tip. More than likely, your login shell is using `/bin/bash` while the provisioner is using `/bin/sh`. *My installs hang when using `apt-get` or `yum`* From fa1f04833c5c8b3c568c87066d906137f3734852 Mon Sep 17 00:00:00 2001 From: Paul Schooss Date: Tue, 20 Jan 2015 11:07:46 -0800 Subject: [PATCH 81/86] Adding documentation for the environments_path --- website/source/docs/provisioners/chef-solo.html.markdown | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/source/docs/provisioners/chef-solo.html.markdown b/website/source/docs/provisioners/chef-solo.html.markdown index b60096307..e4f27f627 100644 --- a/website/source/docs/provisioners/chef-solo.html.markdown +++ b/website/source/docs/provisioners/chef-solo.html.markdown @@ -54,6 +54,10 @@ configuration is actually required, but at least `run_list` is recommended. the secret for encrypted data bags. By default, this is empty, so no secret will be available. +* `environments_path` (string) - The path to the "environments" directory on your local filesystem. + These will be uploaded to the remote machine in the directory specified by the + `staging_directory`. By default, this is empty. + * `execute_command` (string) - The command used to execute Chef. This has various [configuration template variables](/docs/templates/configuration-templates.html) available. See below for more information. From 2038f5da6915d4be3fa3fe82dcebad31292a5dc3 Mon Sep 17 00:00:00 2001 From: Sergio Pantoja Date: Wed, 21 Jan 2015 16:27:34 -0500 Subject: [PATCH 82/86] Fix tutorial source ID `ami-9eaa1cf6` is the correct Ubuntu 14.04 AMI for the us-east-1 region. Fixes #1841 --- website/source/intro/getting-started/build-image.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/intro/getting-started/build-image.html.markdown b/website/source/intro/getting-started/build-image.html.markdown index 9e4a4dedd..1cda08a7c 100644 --- a/website/source/intro/getting-started/build-image.html.markdown +++ b/website/source/intro/getting-started/build-image.html.markdown @@ -54,7 +54,7 @@ briefly. Create a file `example.json` and fill it with the following contents: "access_key": "{{user `aws_access_key`}}", "secret_key": "{{user `aws_secret_key`}}", "region": "us-east-1", - "source_ami": "ami-3d50120d", + "source_ami": "ami-9eaa1cf6", "instance_type": "t2.micro", "ssh_username": "ubuntu", "ami_name": "packer-example {{timestamp}}" From ee6cced42c3228a4047485c676c2ada5e91ea08e Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Thu, 22 Jan 2015 16:06:35 -0500 Subject: [PATCH 83/86] Use the new Travis Docker infrastructure and stuff --- .travis.yml | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4fac7ab93..a22c5ea28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,31 @@ +sudo: false + language: go go: - - 1.2 - - 1.3 - - tip + - 1.2 + - 1.3 + - 1.3 + - tip install: make updatedeps + script: - - GOMAXPROCS=2 make test - #- go test -race ./... + - GOMAXPROCS=2 make test + #- go test -race ./... + +branches: + only: + - master + +notifications: + irc: + channels: + - "irc.freenode.org#packer-tool" + skip_join: true + use_notice: true matrix: - allow_failures: - - go: tip + fast_finish: true + allow_failures: + - go: tip From 1011ec804d464f3b4c05221845f932f708dd677e Mon Sep 17 00:00:00 2001 From: Mehul Kar Date: Thu, 22 Jan 2015 18:48:02 -0800 Subject: [PATCH 84/86] Fix page title --- website/source/docs/command-line/push.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/command-line/push.html.markdown b/website/source/docs/command-line/push.html.markdown index 9717f1142..10165cc22 100644 --- a/website/source/docs/command-line/push.html.markdown +++ b/website/source/docs/command-line/push.html.markdown @@ -5,7 +5,7 @@ description: |- The `packer push` Packer command takes a template and pushes it to a build service that will automatically build this Packer template. --- -# Command-Line: Build +# Command-Line: Push The `packer push` Packer command takes a template and pushes it to a build service. The build service will automatically build your Packer template and From f8d50a9749d48955a93538679a8b4e51bcd19d36 Mon Sep 17 00:00:00 2001 From: Yuya Kusakabe Date: Fri, 23 Jan 2015 11:55:34 +0900 Subject: [PATCH 85/86] Add Travis CI badge to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6104e59d3..46ceb377a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Packer +[![Build Status](https://travis-ci.org/mitchellh/packer.svg?branch=master)](https://travis-ci.org/mitchellh/packer) + * Website: http://www.packer.io * IRC: `#packer-tool` on Freenode * Mailing list: [Google Groups](http://groups.google.com/group/packer-tool) From e3c2f01cb8d8f759c02bd3cfc9d27cc1a941d498 Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Fri, 23 Jan 2015 10:20:57 -0500 Subject: [PATCH 86/86] Test Go 1.4 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a22c5ea28..b0de812ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ language: go go: - 1.2 - 1.3 - - 1.3 + - 1.4 - tip install: make updatedeps