From eef3223f6cee2d1642c42402ab238dd486665671 Mon Sep 17 00:00:00 2001 From: Peter Leschev Date: Thu, 27 Mar 2014 17:11:34 +1100 Subject: [PATCH 01/85] Adding the ability to skip nat port forwarding for ssh connectivity --- builder/virtualbox/common/ssh_config.go | 1 + builder/virtualbox/common/step_export.go | 29 ++++---- builder/virtualbox/common/step_forward_ssh.go | 70 ++++++++++--------- builder/virtualbox/iso/builder.go | 14 ++-- builder/virtualbox/ovf/builder.go | 14 ++-- .../builders/virtualbox-iso.html.markdown | 4 ++ .../builders/virtualbox-ovf.html.markdown | 4 ++ 7 files changed, 80 insertions(+), 56 deletions(-) diff --git a/builder/virtualbox/common/ssh_config.go b/builder/virtualbox/common/ssh_config.go index 00c6167c6..5da8ad403 100644 --- a/builder/virtualbox/common/ssh_config.go +++ b/builder/virtualbox/common/ssh_config.go @@ -16,6 +16,7 @@ type SSHConfig struct { SSHPort uint `mapstructure:"ssh_port"` SSHUser string `mapstructure:"ssh_username"` RawSSHWaitTimeout string `mapstructure:"ssh_wait_timeout"` + SSHSkipNatMapping bool `mapstructure:"ssh_skip_nat_mapping"` SSHWaitTimeout time.Duration } diff --git a/builder/virtualbox/common/step_export.go b/builder/virtualbox/common/step_export.go index e4e860155..0a3cd816c 100644 --- a/builder/virtualbox/common/step_export.go +++ b/builder/virtualbox/common/step_export.go @@ -17,9 +17,10 @@ import ( // Produces: // exportPath string - The path to the resulting export. type StepExport struct { - Format string - OutputDir string - ExportOpts []string + Format string + OutputDir string + ExportOpts []string + SkipNatMapping bool } func (s *StepExport) Run(state multistep.StateBag) multistep.StepAction { @@ -33,15 +34,19 @@ func (s *StepExport) Run(state multistep.StateBag) multistep.StepAction { // Clear out the Packer-created forwarding rule ui.Say("Preparing to export machine...") - ui.Message(fmt.Sprintf( - "Deleting forwarded port mapping for SSH (host port %d)", - state.Get("sshHostPort"))) - command := []string{"modifyvm", vmName, "--natpf1", "delete", "packerssh"} - if err := driver.VBoxManage(command...); err != nil { - err := fmt.Errorf("Error deleting port forwarding rule: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt + var command []string + + if s.SkipNatMapping == false { + ui.Message(fmt.Sprintf( + "Deleting forwarded port mapping for SSH (host port %d)", + state.Get("sshHostPort"))) + command := []string{"modifyvm", vmName, "--natpf1", "delete", "packerssh"} + if err := driver.VBoxManage(command...); err != nil { + err := fmt.Errorf("Error deleting port forwarding rule: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } } // Export the VM to an OVF diff --git a/builder/virtualbox/common/step_forward_ssh.go b/builder/virtualbox/common/step_forward_ssh.go index 862432952..d6d604e00 100644 --- a/builder/virtualbox/common/step_forward_ssh.go +++ b/builder/virtualbox/common/step_forward_ssh.go @@ -19,9 +19,10 @@ import ( // // Produces: type StepForwardSSH struct { - GuestPort uint - HostPortMin uint - HostPortMax uint + GuestPort uint + HostPortMin uint + HostPortMax uint + SkipNatMapping bool } func (s *StepForwardSSH) Run(state multistep.StateBag) multistep.StepAction { @@ -29,39 +30,44 @@ func (s *StepForwardSSH) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) vmName := state.Get("vmName").(string) - log.Printf("Looking for available SSH port between %d and %d", - s.HostPortMin, s.HostPortMax) var sshHostPort uint - var offset uint = 0 + if s.SkipNatMapping { + sshHostPort = s.GuestPort + log.Printf("Skipping SSH NAT mapping and using SSH port %d", sshHostPort) + } else { + log.Printf("Looking for available SSH port between %d and %d", + s.HostPortMin, s.HostPortMax) + var offset uint = 0 - portRange := int(s.HostPortMax - s.HostPortMin) - if portRange > 0 { - // Have to check if > 0 to avoid a panic - offset = uint(rand.Intn(portRange)) - } - - for { - sshHostPort = offset + s.HostPortMin - log.Printf("Trying port: %d", sshHostPort) - l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", sshHostPort)) - if err == nil { - defer l.Close() - break + portRange := int(s.HostPortMax - s.HostPortMin) + if portRange > 0 { + // Have to check if > 0 to avoid a panic + offset = uint(rand.Intn(portRange)) } - } - // Create a forwarded port mapping to the VM - ui.Say(fmt.Sprintf("Creating forwarded port mapping for SSH (host port %d)", sshHostPort)) - command := []string{ - "modifyvm", vmName, - "--natpf1", - fmt.Sprintf("packerssh,tcp,127.0.0.1,%d,,%d", sshHostPort, s.GuestPort), - } - if err := driver.VBoxManage(command...); err != nil { - err := fmt.Errorf("Error creating port forwarding rule: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt + for { + sshHostPort = offset + s.HostPortMin + log.Printf("Trying port: %d", sshHostPort) + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", sshHostPort)) + if err == nil { + defer l.Close() + break + } + } + + // Create a forwarded port mapping to the VM + ui.Say(fmt.Sprintf("Creating forwarded port mapping for SSH (host port %d)", sshHostPort)) + command := []string{ + "modifyvm", vmName, + "--natpf1", + fmt.Sprintf("packerssh,tcp,127.0.0.1,%d,,%d", sshHostPort, s.GuestPort), + } + if err := driver.VBoxManage(command...); err != nil { + err := fmt.Errorf("Error creating port forwarding rule: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } } // Save the port we're using so that future steps can use it diff --git a/builder/virtualbox/iso/builder.go b/builder/virtualbox/iso/builder.go index 31660a961..7ec34bfe8 100644 --- a/builder/virtualbox/iso/builder.go +++ b/builder/virtualbox/iso/builder.go @@ -290,9 +290,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe new(stepAttachGuestAdditions), new(vboxcommon.StepAttachFloppy), &vboxcommon.StepForwardSSH{ - GuestPort: b.config.SSHPort, - HostPortMin: b.config.SSHHostPortMin, - HostPortMax: b.config.SSHHostPortMax, + GuestPort: b.config.SSHPort, + HostPortMin: b.config.SSHHostPortMin, + HostPortMax: b.config.SSHHostPortMax, + SkipNatMapping: b.config.SSHSkipNatMapping, }, &vboxcommon.StepVBoxManage{ Commands: b.config.VBoxManage, @@ -319,9 +320,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, new(vboxcommon.StepRemoveDevices), &vboxcommon.StepExport{ - Format: b.config.Format, - OutputDir: b.config.OutputDir, - ExportOpts: b.config.ExportOpts.ExportOpts, + Format: b.config.Format, + OutputDir: b.config.OutputDir, + ExportOpts: b.config.ExportOpts.ExportOpts, + SkipNatMapping: b.config.SSHSkipNatMapping, }, } diff --git a/builder/virtualbox/ovf/builder.go b/builder/virtualbox/ovf/builder.go index f2e6a92f5..65c4fe72a 100644 --- a/builder/virtualbox/ovf/builder.go +++ b/builder/virtualbox/ovf/builder.go @@ -65,9 +65,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe */ new(vboxcommon.StepAttachFloppy), &vboxcommon.StepForwardSSH{ - GuestPort: b.config.SSHPort, - HostPortMin: b.config.SSHHostPortMin, - HostPortMax: b.config.SSHHostPortMax, + GuestPort: b.config.SSHPort, + HostPortMin: b.config.SSHHostPortMin, + HostPortMax: b.config.SSHHostPortMax, + SkipNatMapping: b.config.SSHSkipNatMapping, }, &vboxcommon.StepVBoxManage{ Commands: b.config.VBoxManage, @@ -95,9 +96,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, new(vboxcommon.StepRemoveDevices), &vboxcommon.StepExport{ - Format: b.config.Format, - OutputDir: b.config.OutputDir, - ExportOpts: b.config.ExportOpts.ExportOpts, + Format: b.config.Format, + OutputDir: b.config.OutputDir, + ExportOpts: b.config.ExportOpts.ExportOpts, + SkipNatMapping: b.config.SSHSkipNatMapping, }, } diff --git a/website/source/docs/builders/virtualbox-iso.html.markdown b/website/source/docs/builders/virtualbox-iso.html.markdown index e6e3b2454..3c2e87e7f 100644 --- a/website/source/docs/builders/virtualbox-iso.html.markdown +++ b/website/source/docs/builders/virtualbox-iso.html.markdown @@ -191,6 +191,10 @@ Optional: available. By default this is "20m", or 20 minutes. Note that this should be quite long since the timer begins as soon as the virtual machine is booted. +* `ssh_skip_nat_mapping` (bool) - Defaults to false. When enabled, Packer does + not setup forwarded port mapping for SSH requests and uses `ssh_port` on the + host to communicate to the virtual machine + * `vboxmanage` (array of array of strings) - Custom `VBoxManage` commands to execute in order to further customize the virtual machine being created. The value of this is an array of commands to execute. The commands are executed diff --git a/website/source/docs/builders/virtualbox-ovf.html.markdown b/website/source/docs/builders/virtualbox-ovf.html.markdown index 6bf6e49d9..43ce70339 100644 --- a/website/source/docs/builders/virtualbox-ovf.html.markdown +++ b/website/source/docs/builders/virtualbox-ovf.html.markdown @@ -126,6 +126,10 @@ Optional: available. By default this is "20m", or 20 minutes. Note that this should be quite long since the timer begins as soon as the virtual machine is booted. +* `ssh_skip_nat_mapping` (bool) - Defaults to false. When enabled, Packer does + not setup forwarded port mapping for SSH requests and uses `ssh_port` on the + host to communicate to the virtual machine + * `vboxmanage` (array of array of strings) - Custom `VBoxManage` commands to execute in order to further customize the virtual machine being created. The value of this is an array of commands to execute. The commands are executed From 42d05368ae45b98548fb327d760ce9447f316c53 Mon Sep 17 00:00:00 2001 From: Colin Hebert Date: Sat, 10 Jan 2015 23:52:45 +1100 Subject: [PATCH 02/85] Save the generated SSH key as a file in debug mode --- builder/digitalocean/builder.go | 5 +++- builder/digitalocean/step_create_ssh_key.go | 31 ++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index cf76f6970..055861984 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -244,7 +244,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ - new(stepCreateSSHKey), + &stepCreateSSHKey{ + Debug: b.config.PackerDebug, + DebugKeyPath: fmt.Sprintf("do_%s.pem", b.config.PackerBuildName), + }, new(stepCreateDroplet), new(stepDropletInfo), &common.StepConnectSSH{ diff --git a/builder/digitalocean/step_create_ssh_key.go b/builder/digitalocean/step_create_ssh_key.go index 78bb474c1..ade19d7b3 100644 --- a/builder/digitalocean/step_create_ssh_key.go +++ b/builder/digitalocean/step_create_ssh_key.go @@ -7,6 +7,8 @@ import ( "encoding/pem" "fmt" "log" + "runtime" + "os" "code.google.com/p/gosshold/ssh" "github.com/mitchellh/multistep" @@ -15,7 +17,9 @@ import ( ) type stepCreateSSHKey struct { - keyId uint + Debug bool + DebugKeyPath string + keyId uint } func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { @@ -62,6 +66,31 @@ func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { // Remember some state for the future state.Put("ssh_key_id", keyId) + // If we're in debug mode, output the private key to the working directory. + if s.Debug { + ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath)) + f, err := os.Create(s.DebugKeyPath) + if err != nil { + state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) + return multistep.ActionHalt + } + defer f.Close() + + // Write the key out + if _, err := f.Write(pem.EncodeToMemory(&priv_blk)); err != nil { + state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) + return multistep.ActionHalt + } + + // Chmod it so that it is SSH ready + if runtime.GOOS != "windows" { + if err := f.Chmod(0600); err != nil { + state.Put("error", fmt.Errorf("Error setting permissions of debug key: %s", err)) + return multistep.ActionHalt + } + } + } + return multistep.ActionContinue } From d046e1cc5b8e1cf00cf8f1d59fc09258abfcca7e Mon Sep 17 00:00:00 2001 From: renat-sabitov-sirca Date: Tue, 16 Dec 2014 12:18:47 +1100 Subject: [PATCH 03/85] Fixing transient AWS errors during EBS builds Relates to #1539 AWS is eventually consistent and instance can be not visibile for some time after creation. This fix eliminates describe-instances call before going to the proper wait loop --- builder/amazon/common/state.go | 6 +++--- .../amazon/common/step_run_source_instance.go | 16 ++++------------ builder/amazon/ebs/step_create_ami.go | 2 +- builder/amazon/ebs/step_stop_instance.go | 2 +- 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/builder/amazon/common/state.go b/builder/amazon/common/state.go index 62e861d74..a3abd6ee9 100644 --- a/builder/amazon/common/state.go +++ b/builder/amazon/common/state.go @@ -63,9 +63,9 @@ func AMIStateRefreshFunc(conn *ec2.EC2, imageId string) StateRefreshFunc { // InstanceStateRefreshFunc returns a StateRefreshFunc that is used to watch // an EC2 instance. -func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc { +func InstanceStateRefreshFunc(conn *ec2.EC2, instanceId string) StateRefreshFunc { return func() (interface{}, string, error) { - resp, err := conn.Instances([]string{i.InstanceId}, ec2.NewFilter()) + resp, err := conn.Instances([]string{instanceId}, ec2.NewFilter()) if err != nil { if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidInstanceID.NotFound" { // Set this to nil as if we didn't find anything. @@ -85,7 +85,7 @@ func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc { return nil, "", nil } - i = &resp.Reservations[0].Instances[0] + i := &resp.Reservations[0].Instances[0] return i, i.State.Name, nil } } diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 50cedf6ea..22e9da847 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -195,21 +195,13 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi instanceId = spotResp.SpotRequestResults[0].InstanceId } - instanceResp, err := ec2conn.Instances([]string{instanceId}, nil) - if err != nil { - err := fmt.Errorf("Error finding source instance (%s): %s", instanceId, err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - s.instance = &instanceResp.Reservations[0].Instances[0] - ui.Message(fmt.Sprintf("Instance ID: %s", s.instance.InstanceId)) + ui.Message(fmt.Sprintf("Instance ID: %s", instanceId)) - ui.Say(fmt.Sprintf("Waiting for instance (%s) to become ready...", s.instance.InstanceId)) + ui.Say(fmt.Sprintf("Waiting for instance (%v) to become ready...", instanceId)) stateChange := StateChangeConf{ Pending: []string{"pending"}, Target: "running", - Refresh: InstanceStateRefreshFunc(ec2conn, s.instance), + Refresh: InstanceStateRefreshFunc(ec2conn, instanceId), StepState: state, } latestInstance, err := WaitForState(&stateChange) @@ -285,7 +277,7 @@ func (s *StepRunSourceInstance) Cleanup(state multistep.StateBag) { } stateChange := StateChangeConf{ Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"}, - Refresh: InstanceStateRefreshFunc(ec2conn, s.instance), + Refresh: InstanceStateRefreshFunc(ec2conn, s.instance.InstanceId), Target: "terminated", } diff --git a/builder/amazon/ebs/step_create_ami.go b/builder/amazon/ebs/step_create_ami.go index f380ea0b1..66390b08e 100644 --- a/builder/amazon/ebs/step_create_ami.go +++ b/builder/amazon/ebs/step_create_ami.go @@ -87,7 +87,7 @@ func (s *stepCreateAMI) Cleanup(state multistep.StateBag) { ui.Error(fmt.Sprintf("Error deregistering AMI, may still be around: %s", err)) return } else if resp.Return == false { - ui.Error(fmt.Sprintf("Error deregistering AMI, may still be around: %t", resp.Return)) + ui.Error(fmt.Sprintf("Error deregistering AMI, may still be around")) return } } diff --git a/builder/amazon/ebs/step_stop_instance.go b/builder/amazon/ebs/step_stop_instance.go index 09c19bddb..ebd0a27f3 100644 --- a/builder/amazon/ebs/step_stop_instance.go +++ b/builder/amazon/ebs/step_stop_instance.go @@ -37,7 +37,7 @@ func (s *stepStopInstance) Run(state multistep.StateBag) multistep.StepAction { stateChange := awscommon.StateChangeConf{ Pending: []string{"running", "stopping"}, Target: "stopped", - Refresh: awscommon.InstanceStateRefreshFunc(ec2conn, instance), + Refresh: awscommon.InstanceStateRefreshFunc(ec2conn, instance.InstanceId), StepState: state, } _, err = awscommon.WaitForState(&stateChange) From a7eeb6a6a796690af4c84d0ee6cda0abc48fbc8f Mon Sep 17 00:00:00 2001 From: Brandon Heller Date: Thu, 4 Jun 2015 02:43:16 -0700 Subject: [PATCH 04/85] vmware/iso: support hierarchical output directories When providing a hierarchical output_directory value like 'transient/jenkins-slave', the VM would fail to build in the CreateDisk step. The properly created output directory would not match the location provided to CreateDisk, since datastorePath() did not properly split such paths. Now this case works; tested hierarchical and singular output_directory values. --- builder/vmware/iso/driver_esx5.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/vmware/iso/driver_esx5.go b/builder/vmware/iso/driver_esx5.go index 2b9332760..0d6fd6531 100644 --- a/builder/vmware/iso/driver_esx5.go +++ b/builder/vmware/iso/driver_esx5.go @@ -311,8 +311,8 @@ func (d *ESX5Driver) String() string { } func (d *ESX5Driver) datastorePath(path string) string { - baseDir := filepath.Base(filepath.Dir(path)) - return filepath.ToSlash(filepath.Join("/vmfs/volumes", d.Datastore, baseDir, filepath.Base(path))) + dirPath := filepath.Dir(path) + return filepath.ToSlash(filepath.Join("/vmfs/volumes", d.Datastore, dirPath, filepath.Base(path))) } func (d *ESX5Driver) cachePath(path string) string { From fab9ca9cdbbefd236c10c351d57cba940fcc66b0 Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Tue, 4 Mar 2014 13:23:07 -0500 Subject: [PATCH 05/85] Initial work to implement additional disk support in the vmware-iso builder * Matches the syntax from mitchellh/packer#703 * Creates disk(s), adds them to the vmx template, and runs compact at the end --- builder/vmware/common/step_compact_disk.go | 12 +++++++++ builder/vmware/iso/builder.go | 1 + builder/vmware/iso/step_create_disk.go | 22 ++++++++++++++++ builder/vmware/iso/step_create_vmx.go | 30 ++++++++++++++++++++++ 4 files changed, 65 insertions(+) diff --git a/builder/vmware/common/step_compact_disk.go b/builder/vmware/common/step_compact_disk.go index 777e8aaea..a76cf4d7f 100644 --- a/builder/vmware/common/step_compact_disk.go +++ b/builder/vmware/common/step_compact_disk.go @@ -36,6 +36,18 @@ func (s StepCompactDisk) Run(state multistep.StateBag) multistep.StepAction { state.Put("error", fmt.Errorf("Error compacting disk: %s", err)) return multistep.ActionHalt } + + moreDisks := state.Get("additional_disk_paths").([]string) + if len(moreDisks) > 0 { + for i, path := range moreDisks { + ui.Say(fmt.Sprintf("Compacting additional disk image %d",i+1)) + if err := driver.CompactDisk(path); err != nil { + state.Put("error", fmt.Errorf("Error compacting additional disk %d: %s", i+1, err)) + return multistep.ActionHalt + } + } + + } return multistep.ActionContinue } diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go index c63bfdc8b..065d20603 100644 --- a/builder/vmware/iso/builder.go +++ b/builder/vmware/iso/builder.go @@ -35,6 +35,7 @@ type Config struct { vmwcommon.ToolsConfig `mapstructure:",squash"` vmwcommon.VMXConfig `mapstructure:",squash"` + AdditionalDiskSize []uint `mapstructure:"additionaldisk_size"` DiskName string `mapstructure:"vmdk_name"` DiskSize uint `mapstructure:"disk_size"` DiskTypeId string `mapstructure:"disk_type_id"` diff --git a/builder/vmware/iso/step_create_disk.go b/builder/vmware/iso/step_create_disk.go index b357cad01..b5a1bfd87 100644 --- a/builder/vmware/iso/step_create_disk.go +++ b/builder/vmware/iso/step_create_disk.go @@ -34,6 +34,28 @@ func (stepCreateDisk) Run(state multistep.StateBag) multistep.StepAction { } state.Put("full_disk_path", full_disk_path) + + if len(config.AdditionalDiskSize) > 0 { + // stash the disk paths we create + additional_paths := make([]string,len(config.AdditionalDiskSize)) + + ui.Say("Creating additional hard drives...") + for i, additionalsize := range config.AdditionalDiskSize { + additionalpath := filepath.Join(config.OutputDir, fmt.Sprintf("%s-%d.vmdk",config.DiskName,i+1)) + size := fmt.Sprintf("%dM", uint64(additionalsize)) + + if err := driver.CreateDisk(additionalpath, size, config.DiskTypeId); err != nil { + err := fmt.Errorf("Error creating additional disk: %s", err) + state.Put("error",err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + additional_paths[i] = additionalpath + } + + state.Put("additional_disk_paths", additional_paths) + } return multistep.ActionContinue } diff --git a/builder/vmware/iso/step_create_vmx.go b/builder/vmware/iso/step_create_vmx.go index 5a1a829b5..dde3859a0 100644 --- a/builder/vmware/iso/step_create_vmx.go +++ b/builder/vmware/iso/step_create_vmx.go @@ -20,6 +20,11 @@ type vmxTemplateData struct { Version string } +type additionalDiskTemplateData struct { + DiskNumber int + DiskName string +} + // This step creates the VMX file for the VM. // // Uses: @@ -70,6 +75,25 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { vmxTemplate = string(rawBytes) } + + if len(config.AdditionalDiskSize) > 0 { + for i, _ := range config.AdditionalDiskSize { + data := &additionalDiskTemplateData{ + DiskNumber: i+1, + DiskName: config.DiskName, + } + + diskTemplate, err := config.tpl.Process(DefaultAdditionalDiskTemplate,data) + if err != nil { + err := fmt.Errorf("Error preparing VMX template for additional disk: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + vmxTemplate += diskTemplate + } + } vmxContents, err := interpolate.Render(vmxTemplate, &ctx) if err != nil { @@ -191,3 +215,9 @@ vmci0.pciSlotNumber = "35" vmci0.present = "TRUE" vmotion.checkpointFBSize = "65536000" ` + +const DefaultAdditionalDiskTemplate = ` +scsi0:{{ .DiskNumber }}.fileName = "{{ .DiskName}}-{{ .DiskNumber }}.vmdk" +scsi0:{{ .DiskNumber }}.present = "TRUE" +scsi0:{{ .DiskNumber }}.redo = "" +` From 7dfb837ddb91fa0cd8562b5d16bd84a29c5b2ad7 Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Tue, 4 Mar 2014 15:00:24 -0500 Subject: [PATCH 06/85] Formatting cleanups from `go fmt` --- builder/vmware/iso/step_create_disk.go | 16 ++++++++-------- builder/vmware/iso/step_create_vmx.go | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/builder/vmware/iso/step_create_disk.go b/builder/vmware/iso/step_create_disk.go index b5a1bfd87..ded3a86da 100644 --- a/builder/vmware/iso/step_create_disk.go +++ b/builder/vmware/iso/step_create_disk.go @@ -34,26 +34,26 @@ func (stepCreateDisk) Run(state multistep.StateBag) multistep.StepAction { } state.Put("full_disk_path", full_disk_path) - + if len(config.AdditionalDiskSize) > 0 { // stash the disk paths we create - additional_paths := make([]string,len(config.AdditionalDiskSize)) - + additional_paths := make([]string, len(config.AdditionalDiskSize)) + ui.Say("Creating additional hard drives...") for i, additionalsize := range config.AdditionalDiskSize { - additionalpath := filepath.Join(config.OutputDir, fmt.Sprintf("%s-%d.vmdk",config.DiskName,i+1)) + additionalpath := filepath.Join(config.OutputDir, fmt.Sprintf("%s-%d.vmdk", config.DiskName, i+1)) size := fmt.Sprintf("%dM", uint64(additionalsize)) - + if err := driver.CreateDisk(additionalpath, size, config.DiskTypeId); err != nil { err := fmt.Errorf("Error creating additional disk: %s", err) - state.Put("error",err) + state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - + additional_paths[i] = additionalpath } - + state.Put("additional_disk_paths", additional_paths) } diff --git a/builder/vmware/iso/step_create_vmx.go b/builder/vmware/iso/step_create_vmx.go index dde3859a0..97cd1262c 100644 --- a/builder/vmware/iso/step_create_vmx.go +++ b/builder/vmware/iso/step_create_vmx.go @@ -21,8 +21,8 @@ type vmxTemplateData struct { } type additionalDiskTemplateData struct { - DiskNumber int - DiskName string + DiskNumber int + DiskName string } // This step creates the VMX file for the VM. @@ -75,22 +75,22 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { vmxTemplate = string(rawBytes) } - + if len(config.AdditionalDiskSize) > 0 { for i, _ := range config.AdditionalDiskSize { data := &additionalDiskTemplateData{ - DiskNumber: i+1, - DiskName: config.DiskName, + DiskNumber: i + 1, + DiskName: config.DiskName, } - - diskTemplate, err := config.tpl.Process(DefaultAdditionalDiskTemplate,data) + + diskTemplate, err := config.tpl.Process(DefaultAdditionalDiskTemplate, data) if err != nil { err := fmt.Errorf("Error preparing VMX template for additional disk: %s", err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - + vmxTemplate += diskTemplate } } From 5f183026b337d8b64f5a50a20931de457a8ee023 Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Tue, 4 Mar 2014 15:48:15 -0500 Subject: [PATCH 07/85] Fix vmware compact_disk step when there are no additional disks --- builder/vmware/common/step_compact_disk.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/builder/vmware/common/step_compact_disk.go b/builder/vmware/common/step_compact_disk.go index a76cf4d7f..4319b7fe9 100644 --- a/builder/vmware/common/step_compact_disk.go +++ b/builder/vmware/common/step_compact_disk.go @@ -37,16 +37,16 @@ func (s StepCompactDisk) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - moreDisks := state.Get("additional_disk_paths").([]string) - if len(moreDisks) > 0 { - for i, path := range moreDisks { - ui.Say(fmt.Sprintf("Compacting additional disk image %d",i+1)) - if err := driver.CompactDisk(path); err != nil { - state.Put("error", fmt.Errorf("Error compacting additional disk %d: %s", i+1, err)) - return multistep.ActionHalt + if state.Get("additional_disk_paths") != nil { + if moreDisks := state.Get("additional_disk_paths").([]string); len(moreDisks) > 0 { + for i, path := range moreDisks { + ui.Say(fmt.Sprintf("Compacting additional disk image %d",i+1)) + if err := driver.CompactDisk(path); err != nil { + state.Put("error", fmt.Errorf("Error compacting additional disk %d: %s", i+1, err)) + return multistep.ActionHalt + } } } - } return multistep.ActionContinue From b665339b39f007a2e3121a99eae959487b4208b1 Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Mon, 15 Sep 2014 09:47:26 -0700 Subject: [PATCH 08/85] Added website docs for VMWare ISO additionaldisk_size --- website/source/docs/builders/vmware-iso.html.markdown | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/website/source/docs/builders/vmware-iso.html.markdown b/website/source/docs/builders/vmware-iso.html.markdown index b1d026e44..b021ca532 100644 --- a/website/source/docs/builders/vmware-iso.html.markdown +++ b/website/source/docs/builders/vmware-iso.html.markdown @@ -72,6 +72,12 @@ each category, the available options are alphabetized and described. ### Optional: +* `additionaldisk_size` (array of integers) - The size(s) of any additional + hard disks for the VM in megabytes. If this is not specified then the VM will + only contain a primary hard disk. The builder uses expandable, not fixed-size + virtual hard disks, so the actual file representing the disk will not use the + full size unless it is full. + * `boot_command` (array of strings) - This is an array of commands to type when the virtual machine is first booted. The goal of these commands should be to type just enough to initialize the operating system installer. Special From e9a491ae45871c42bf9b86c302b6ed26657b8acf Mon Sep 17 00:00:00 2001 From: Shawn Neal Date: Thu, 4 Jun 2015 11:44:07 -0700 Subject: [PATCH 09/85] New interpolation for additional vmware disks --- builder/vmware/iso/step_create_vmx.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/builder/vmware/iso/step_create_vmx.go b/builder/vmware/iso/step_create_vmx.go index 97cd1262c..69cb3f261 100644 --- a/builder/vmware/iso/step_create_vmx.go +++ b/builder/vmware/iso/step_create_vmx.go @@ -45,15 +45,6 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { ui.Say("Building and writing VMX file") - ctx := config.ctx - ctx.Data = &vmxTemplateData{ - Name: config.VMName, - GuestOS: config.GuestOSType, - DiskName: config.DiskName, - Version: config.Version, - ISOPath: isoPath, - } - vmxTemplate := DefaultVMXTemplate if config.VMXTemplatePath != "" { f, err := os.Open(config.VMXTemplatePath) @@ -76,14 +67,16 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { vmxTemplate = string(rawBytes) } + ctx := config.ctx + if len(config.AdditionalDiskSize) > 0 { for i, _ := range config.AdditionalDiskSize { - data := &additionalDiskTemplateData{ + ctx.Data = &additionalDiskTemplateData{ DiskNumber: i + 1, DiskName: config.DiskName, } - diskTemplate, err := config.tpl.Process(DefaultAdditionalDiskTemplate, data) + diskTemplate, err := interpolate.Render(DefaultAdditionalDiskTemplate, &ctx) if err != nil { err := fmt.Errorf("Error preparing VMX template for additional disk: %s", err) state.Put("error", err) @@ -95,6 +88,14 @@ func (s *stepCreateVMX) Run(state multistep.StateBag) multistep.StepAction { } } + ctx.Data = &vmxTemplateData{ + Name: config.VMName, + GuestOS: config.GuestOSType, + DiskName: config.DiskName, + Version: config.Version, + ISOPath: isoPath, + } + vmxContents, err := interpolate.Render(vmxTemplate, &ctx) if err != nil { err := fmt.Errorf("Error procesing VMX template: %s", err) From e65e2d104afa8ed5e1846579c9bede7676d5518b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 20:41:39 -0700 Subject: [PATCH 10/85] common: StepDownload can force an extension --- common/step_download.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/common/step_download.go b/common/step_download.go index 8d6378adc..b8bd60b5e 100644 --- a/common/step_download.go +++ b/common/step_download.go @@ -1,6 +1,7 @@ package common import ( + "crypto/sha1" "encoding/hex" "fmt" "log" @@ -36,6 +37,12 @@ type StepDownload struct { // A list of URLs to attempt to download this thing. Url []string + + // Extension is the extension to force for the file that is downloaded. + // Some systems require a certain extension. If this isn't set, the + // extension on the URL is used. Otherwise, this will be forced + // on the downloaded file for every URL. + Extension string } func (s *StepDownload) Run(state multistep.StateBag) multistep.StepAction { @@ -60,9 +67,19 @@ func (s *StepDownload) Run(state multistep.StateBag) multistep.StepAction { targetPath := s.TargetPath if targetPath == "" { + // Determine a cache key. This is normally just the URL but + // if we force a certain extension we hash the URL and add + // the extension to force it. + cacheKey := url + if s.Extension != "" { + hash := sha1.Sum([]byte(url)) + cacheKey = fmt.Sprintf( + "%s.%s", hex.EncodeToString(hash[:]), s.Extension) + } + log.Printf("Acquiring lock to download: %s", url) - targetPath = cache.Lock(url) - defer cache.Unlock(url) + targetPath = cache.Lock(cacheKey) + defer cache.Unlock(cacheKey) } config := &DownloadConfig{ From 9ea34d4ea85d469272a12dc0ab6fe6837128e382 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Mon, 8 Jun 2015 20:42:16 -0700 Subject: [PATCH 11/85] virtualbox/iso: force iso extension for downloads --- builder/virtualbox/iso/builder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/virtualbox/iso/builder.go b/builder/virtualbox/iso/builder.go index d095eae2d..b6ca982fd 100644 --- a/builder/virtualbox/iso/builder.go +++ b/builder/virtualbox/iso/builder.go @@ -230,6 +230,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Description: "ISO", ResultKey: "iso_path", Url: b.config.ISOUrls, + Extension: "iso", }, &vboxcommon.StepOutputDir{ Force: b.config.PackerForce, From 999b0874cce88279a3d39dce441a342e7c5f5017 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Tue, 9 Jun 2015 11:38:53 -0500 Subject: [PATCH 12/85] Update AWS EBS builder to fix invalid params --- builder/amazon/common/block_device.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/block_device.go b/builder/amazon/common/block_device.go index 14add3276..c44d03561 100644 --- a/builder/amazon/common/block_device.go +++ b/builder/amazon/common/block_device.go @@ -29,13 +29,23 @@ func buildBlockDevices(b []BlockDevice) []*ec2.BlockDeviceMapping { for _, blockDevice := range b { ebsBlockDevice := &ec2.EBSBlockDevice{ - SnapshotID: &blockDevice.SnapshotId, - Encrypted: &blockDevice.Encrypted, - IOPS: &blockDevice.IOPS, VolumeType: &blockDevice.VolumeType, VolumeSize: &blockDevice.VolumeSize, DeleteOnTermination: &blockDevice.DeleteOnTermination, } + + // IOPS is only valid for SSD Volumes + if blockDevice.VolumeType != "standard" && blockDevice.VolumeType != "gp2" { + ebsBlockDevice.IOPS = &blockDevice.IOPS + } + + // You cannot specify Encrypted if you specify a Snapshot ID + if blockDevice.SnapshotId != "" { + ebsBlockDevice.SnapshotID = &blockDevice.SnapshotId + } else { + ebsBlockDevice.Encrypted = &blockDevice.Encrypted + } + mapping := &ec2.BlockDeviceMapping{ EBS: ebsBlockDevice, DeviceName: &blockDevice.DeviceName, From 4da118c64f22b950cb596c5c370751be5f59121b Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Tue, 9 Jun 2015 11:56:40 -0500 Subject: [PATCH 13/85] fix up tests --- builder/amazon/common/block_device_test.go | 27 +++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/block_device_test.go b/builder/amazon/common/block_device_test.go index c4f644f67..aacd54cb4 100644 --- a/builder/amazon/common/block_device_test.go +++ b/builder/amazon/common/block_device_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awsutil" "github.com/aws/aws-sdk-go/service/ec2" ) @@ -28,11 +29,31 @@ func TestBlockDevice(t *testing.T) { DeviceName: aws.String("/dev/sdb"), VirtualName: aws.String("ephemeral0"), EBS: &ec2.EBSBlockDevice{ - Encrypted: aws.Boolean(false), SnapshotID: aws.String("snap-1234"), VolumeType: aws.String("standard"), VolumeSize: aws.Long(8), DeleteOnTermination: aws.Boolean(true), + }, + }, + }, + { + Config: &BlockDevice{ + DeviceName: "/dev/sdb", + VirtualName: "ephemeral0", + VolumeType: "io1", + VolumeSize: 8, + DeleteOnTermination: true, + IOPS: 1000, + }, + + Result: &ec2.BlockDeviceMapping{ + DeviceName: aws.String("/dev/sdb"), + VirtualName: aws.String("ephemeral0"), + EBS: &ec2.EBSBlockDevice{ + Encrypted: aws.Boolean(false), + VolumeType: aws.String("io1"), + VolumeSize: aws.Long(8), + DeleteOnTermination: aws.Boolean(true), IOPS: aws.Long(1000), }, }, @@ -48,11 +69,11 @@ func TestBlockDevice(t *testing.T) { expected := []*ec2.BlockDeviceMapping{tc.Result} got := blockDevices.BuildAMIDevices() if !reflect.DeepEqual(expected, got) { - t.Fatalf("bad: %#v", expected) + t.Fatalf("Bad block device, \nexpected: %s\n\ngot: %s", awsutil.StringValue(expected), awsutil.StringValue(got)) } if !reflect.DeepEqual(expected, blockDevices.BuildLaunchDevices()) { - t.Fatalf("bad: %#v", expected) + t.Fatalf("Bad block device, \nexpected: %s\n\ngot: %s", awsutil.StringValue(expected), awsutil.StringValue(blockDevices.BuildLaunchDevices())) } } } From 91cfe5edf45404dc44b7decb0854baa3f2cc7236 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Jun 2015 20:57:32 -0700 Subject: [PATCH 14/85] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 027fc6fed..26f0274a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,8 @@ BUG FIXES: OS installers. [GH-1709] * builder/virtualbox: Remove the floppy controller in addition to the floppy disk. [GH-1879] + * builder/virtualbox: Fixed regression where downloading ISO without a + ".iso" extension didn't work. [GH-1839] * builder/vmware: Add 100ms delay between keystrokes to avoid subtle timing issues in most cases. [GH-1663] * builder/vmware: Bind HTTP server to IPv4, which is more compatible with From 6e9fb6a9d1c019da4d9b442b73b46312c7c91d87 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Jun 2015 21:09:09 -0700 Subject: [PATCH 15/85] command/push: the -name parameter actually works --- command/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/push.go b/command/push.go index 6c04917dd..b4007db80 100644 --- a/command/push.go +++ b/command/push.go @@ -157,7 +157,7 @@ func (c *PushCommand) Run(args []string) int { // Build the upload options var uploadOpts uploadOpts - uploadOpts.Slug = push.Name + uploadOpts.Slug = name uploadOpts.Builds = make(map[string]*uploadBuildInfo) for _, b := range tpl.Builders { info := &uploadBuildInfo{Type: b.Type} From 2a6c4e0d2c2ee20b3715691ca0ed1934c71e3bec Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Tue, 9 Jun 2015 21:09:56 -0700 Subject: [PATCH 16/85] command/push: output fix --- command/push.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/push.go b/command/push.go index b4007db80..f888186de 100644 --- a/command/push.go +++ b/command/push.go @@ -236,7 +236,7 @@ func (c *PushCommand) Run(args []string) int { return 1 } - c.Ui.Say(fmt.Sprintf("Push successful to '%s'", push.Name)) + c.Ui.Say(fmt.Sprintf("Push successful to '%s'", name)) return 0 } From 8393b85ed579dd2b6a04325c8e5e7c14ed1610d6 Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Wed, 10 Jun 2015 07:41:12 +0200 Subject: [PATCH 17/85] Support Parallels Desktop 11 --- builder/parallels/common/driver.go | 6 ++++++ builder/parallels/common/driver_10.go | 1 + 2 files changed, 7 insertions(+) diff --git a/builder/parallels/common/driver.go b/builder/parallels/common/driver.go index bdb73719c..5c3d93c09 100644 --- a/builder/parallels/common/driver.go +++ b/builder/parallels/common/driver.go @@ -73,6 +73,12 @@ func NewDriver() (Driver, error) { log.Printf("prlctl path: %s", prlctlPath) drivers = map[string]Driver{ + "11": &Parallels10Driver{ + Parallels9Driver: Parallels9Driver{ + PrlctlPath: prlctlPath, + dhcp_lease_file: dhcp_lease_file, + }, + }, "10": &Parallels10Driver{ Parallels9Driver: Parallels9Driver{ PrlctlPath: prlctlPath, diff --git a/builder/parallels/common/driver_10.go b/builder/parallels/common/driver_10.go index ae39240f4..9ab0754de 100644 --- a/builder/parallels/common/driver_10.go +++ b/builder/parallels/common/driver_10.go @@ -1,6 +1,7 @@ package common // Parallels10Driver are inherited from Parallels9Driver. +// Used for Parallels v 10 & 11 type Parallels10Driver struct { Parallels9Driver } From 5d600c70f2356c7b5d8a06eb363e9c05ea824a77 Mon Sep 17 00:00:00 2001 From: Jan Schumann Date: Wed, 10 Jun 2015 17:20:41 +0200 Subject: [PATCH 18/85] added test to illustrate wrong behavior --- builder/amazon/common/block_device_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/builder/amazon/common/block_device_test.go b/builder/amazon/common/block_device_test.go index aacd54cb4..12d1530bf 100644 --- a/builder/amazon/common/block_device_test.go +++ b/builder/amazon/common/block_device_test.go @@ -36,6 +36,23 @@ func TestBlockDevice(t *testing.T) { }, }, }, + { + Config: &BlockDevice{ + DeviceName: "/dev/sdb", + VolumeSize: 8, + }, + + Result: &ec2.BlockDeviceMapping{ + DeviceName: aws.String("/dev/sdb"), + VirtualName: aws.String(""), + EBS: &ec2.EBSBlockDevice{ + Encrypted: aws.Boolean(false), + VolumeType: aws.String(""), + VolumeSize: aws.Long(8), + DeleteOnTermination: aws.Boolean(false), + }, + }, + }, { Config: &BlockDevice{ DeviceName: "/dev/sdb", From 802cfa38715bd12851f2e8719c5165338c78f6ed Mon Sep 17 00:00:00 2001 From: Jan Schumann Date: Wed, 10 Jun 2015 17:21:25 +0200 Subject: [PATCH 19/85] make sure IOPS is not set for empty VolumeType --- builder/amazon/common/block_device.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/amazon/common/block_device.go b/builder/amazon/common/block_device.go index c44d03561..e97cd4107 100644 --- a/builder/amazon/common/block_device.go +++ b/builder/amazon/common/block_device.go @@ -35,7 +35,7 @@ func buildBlockDevices(b []BlockDevice) []*ec2.BlockDeviceMapping { } // IOPS is only valid for SSD Volumes - if blockDevice.VolumeType != "standard" && blockDevice.VolumeType != "gp2" { + if blockDevice.VolumeType != "" && blockDevice.VolumeType != "standard" && blockDevice.VolumeType != "gp2" { ebsBlockDevice.IOPS = &blockDevice.IOPS } From 7410289cbad45e6b8f15aee27aef0ab8443d9869 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:21:32 -0700 Subject: [PATCH 20/85] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26f0274a2..39f222fae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] + * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear * buidler/openstakc: Add `ssh_interface` option for rackconnect for users that From be4a82dfae69f990092db29a24749a43cf3c3dfb Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:33:01 -0700 Subject: [PATCH 21/85] amazon/*: fix some merge conflicts --- builder/amazon/common/state.go | 2 +- builder/amazon/common/step_run_source_instance.go | 2 +- builder/amazon/ebs/step_stop_instance.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/state.go b/builder/amazon/common/state.go index 9acedd020..075ce8ef7 100644 --- a/builder/amazon/common/state.go +++ b/builder/amazon/common/state.go @@ -91,7 +91,7 @@ func InstanceStateRefreshFunc(conn *ec2.EC2, instanceId string) StateRefreshFunc return nil, "", nil } - i = resp.Reservations[0].Instances[0] + i := resp.Reservations[0].Instances[0] return i, *i.State.Name, nil } } diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 87ac077b9..92dafa564 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -310,7 +310,7 @@ func (s *StepRunSourceInstance) Cleanup(state multistep.StateBag) { } stateChange := StateChangeConf{ Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"}, - Refresh: InstanceStateRefreshFunc(ec2conn, s.instance.InstanceId), + Refresh: InstanceStateRefreshFunc(ec2conn, *s.instance.InstanceID), Target: "terminated", } diff --git a/builder/amazon/ebs/step_stop_instance.go b/builder/amazon/ebs/step_stop_instance.go index 7312fd200..967e5bbf4 100644 --- a/builder/amazon/ebs/step_stop_instance.go +++ b/builder/amazon/ebs/step_stop_instance.go @@ -40,7 +40,7 @@ func (s *stepStopInstance) Run(state multistep.StateBag) multistep.StepAction { stateChange := awscommon.StateChangeConf{ Pending: []string{"running", "stopping"}, Target: "stopped", - Refresh: awscommon.InstanceStateRefreshFunc(ec2conn, instance.InstanceId), + Refresh: awscommon.InstanceStateRefreshFunc(ec2conn, *instance.InstanceID), StepState: state, } _, err = awscommon.WaitForState(&stateChange) From 77c8df1e7b3b8910e8fb60d6f8ad37d57d4e417f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:33:30 -0700 Subject: [PATCH 22/85] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39f222fae..7542c7b97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ BUG FIXES: is deleted on cleanup. [GH-1801] * builder/amazon: AMI copy won't copy to the source region [GH-2123] * builder/amazon: Validate AMI doesn't exist with name prior to build [GH-1774] + * builder/amazon: Improved retry logic around waiting for instances. [GH-1764] * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] * builder/digitalocean: Ignore invalid fields from the ever-changing v2 API From bd04b52b323472922d0fa1dbec9a870296179836 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:50:08 -0700 Subject: [PATCH 23/85] virtualbox/common: style --- builder/virtualbox/common/step_export.go | 9 +++------ builder/virtualbox/common/step_forward_ssh.go | 7 ++----- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/builder/virtualbox/common/step_export.go b/builder/virtualbox/common/step_export.go index 0a3cd816c..56b013d9f 100644 --- a/builder/virtualbox/common/step_export.go +++ b/builder/virtualbox/common/step_export.go @@ -31,12 +31,10 @@ func (s *StepExport) Run(state multistep.StateBag) multistep.StepAction { // Wait a second to ensure VM is really shutdown. log.Println("1 second timeout to ensure VM is really shutdown") time.Sleep(1 * time.Second) + ui.Say("Preparing to export machine...") // Clear out the Packer-created forwarding rule - ui.Say("Preparing to export machine...") - var command []string - - if s.SkipNatMapping == false { + if !s.SkipNatMapping { ui.Message(fmt.Sprintf( "Deleting forwarded port mapping for SSH (host port %d)", state.Get("sshHostPort"))) @@ -52,13 +50,12 @@ func (s *StepExport) Run(state multistep.StateBag) multistep.StepAction { // Export the VM to an OVF outputPath := filepath.Join(s.OutputDir, vmName+"."+s.Format) - command = []string{ + command := []string{ "export", vmName, "--output", outputPath, } - command = append(command, s.ExportOpts...) ui.Say("Exporting virtual machine...") diff --git a/builder/virtualbox/common/step_forward_ssh.go b/builder/virtualbox/common/step_forward_ssh.go index d6d604e00..fe6004281 100644 --- a/builder/virtualbox/common/step_forward_ssh.go +++ b/builder/virtualbox/common/step_forward_ssh.go @@ -30,11 +30,8 @@ func (s *StepForwardSSH) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) vmName := state.Get("vmName").(string) - var sshHostPort uint - if s.SkipNatMapping { - sshHostPort = s.GuestPort - log.Printf("Skipping SSH NAT mapping and using SSH port %d", sshHostPort) - } else { + sshHostPort := s.GuestPort + if !s.SkipNatMapping { log.Printf("Looking for available SSH port between %d and %d", s.HostPortMin, s.HostPortMax) var offset uint = 0 From 9b83b7f2e05fd3a83ee65b53eb05aab1f4786527 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 10:50:40 -0700 Subject: [PATCH 24/85] update CHANGELOg --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7542c7b97..a98e0bedc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ IMPROVEMENTS: RackConnect data to appear * buidler/openstakc: Add `ssh_interface` option for rackconnect for users that have prohibitive firewalls + * builder/virtualbox: Added option: `ssh_skip_nat_mapping` to skip the + automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support * command/push: Add `-name` flag for specifying name from CLI [GH-2042] * command/push: Push configuration in templates supports variables [GH-1861] From 626243baed02592f76088c3e5ae7140e73d82b5b Mon Sep 17 00:00:00 2001 From: Clint Date: Wed, 10 Jun 2015 13:18:17 -0500 Subject: [PATCH 25/85] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a98e0bedc..a6f179461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] + * builder/amazon-ebs: Add pre-build step, validate unique AMI name [GH-2187] * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear From 0ff508c80d57447c740830a968eaff32cf42257b Mon Sep 17 00:00:00 2001 From: Clint Date: Wed, 10 Jun 2015 13:20:39 -0500 Subject: [PATCH 26/85] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6f179461..b0abd0cb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] - * builder/amazon-ebs: Add pre-build step, validate unique AMI name [GH-2187] * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear @@ -38,6 +37,7 @@ BUG FIXES: * builder/amazon: Improved retry logic around waiting for instances. [GH-1764] * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] + * builder/amazon/instance: Use `--region` flag for bundle upload command. [GH-1931] * builder/digitalocean: Ignore invalid fields from the ever-changing v2 API * builder/digitalocean: Private images can be used as a source [GH-1792] * builder/docker: Fixed hang on prompt while copying script From 83fc70fa00d84cee46399bef2ff8cdaeddfa49bf Mon Sep 17 00:00:00 2001 From: Clint Date: Wed, 10 Jun 2015 13:22:36 -0500 Subject: [PATCH 27/85] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0abd0cb9..8c9a2c151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ BUG FIXES: * builder/amazon: AMI copy won't copy to the source region [GH-2123] * builder/amazon: Validate AMI doesn't exist with name prior to build [GH-1774] * builder/amazon: Improved retry logic around waiting for instances. [GH-1764] + * builder/amazon: Fix issues with creating Block Devices. [GH-2195] * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] * builder/amazon/instance: Use `--region` flag for bundle upload command. [GH-1931] From fafdfc962f7272afe390961e804de4fe67879f85 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 11:31:20 -0700 Subject: [PATCH 28/85] vmware/common: detect Vmware 'unknown error' and show better message --- builder/vmware/common/driver.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/builder/vmware/common/driver.go b/builder/vmware/common/driver.go index c645f40b7..ee8dbc30e 100644 --- a/builder/vmware/common/driver.go +++ b/builder/vmware/common/driver.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os/exec" + "regexp" "runtime" "strconv" "strings" @@ -135,6 +136,18 @@ func runAndLog(cmd *exec.Cmd) (string, string, error) { } err = fmt.Errorf("VMware error: %s", message) + + // If "unknown error" is in there, add some additional notes + re := regexp.MustCompile(`(?i)unknown error`) + if re.MatchString(message) { + err = fmt.Errorf( + "%s\n\n%s", err, + "Packer detected a VMware 'Unknown Error'. Unfortunately VMware\n"+ + "often has extremely vague error messages such as this and Packer\n"+ + "itself can't do much about that. Please check the vmware.log files\n"+ + "created by VMware when a VM is started (in the directory of the\n"+ + "vmx file), which often contains more detailed error information.") + } } log.Printf("stdout: %s", stdoutString) From 802a76685ee14cbd1ece648f8721d7564287c831 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 11:34:37 -0700 Subject: [PATCH 29/85] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c9a2c151..86e1cbdb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ BUG FIXES: OS installers. [GH-1709] * builder/vmware: Case-insensitive match of MAC address to find IP [GH-1989] * builder/vmware: More robust IP parsing from ifconfig output [GH-1999] + * builder/vmware: Nested output directories for ESXi work [GH-2174] * command/validate: don't crash for invalid builds [GH-2139] * post-processor/atlas: Find common archive prefix for Windows [GH-1874] * post-processor/atlas: Fix index out of range panic [GH-1959] From ffdd5e6772f2785e4d8ee3ea97d7438a76c1892e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 12:40:20 -0700 Subject: [PATCH 30/85] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86e1cbdb1..3a33dbc92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ IMPROVEMENTS: * builder/virtualbox: Added option: `ssh_skip_nat_mapping` to skip the automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support + * builder/vmware: Support for additional disks [GH-1382] * command/push: Add `-name` flag for specifying name from CLI [GH-2042] * command/push: Push configuration in templates supports variables [GH-1861] * post-processor/docker-save: Can be chained [GH-2179] From 9da9ce6046b3c40a9fe5ac09a38aea6661325d9d Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 12:41:17 -0700 Subject: [PATCH 31/85] vmware/iso: disk_additional_size --- builder/vmware/iso/builder.go | 28 +++++++++---------- .../docs/builders/vmware-iso.html.markdown | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/builder/vmware/iso/builder.go b/builder/vmware/iso/builder.go index 065d20603..352d02960 100644 --- a/builder/vmware/iso/builder.go +++ b/builder/vmware/iso/builder.go @@ -35,20 +35,20 @@ type Config struct { vmwcommon.ToolsConfig `mapstructure:",squash"` vmwcommon.VMXConfig `mapstructure:",squash"` - AdditionalDiskSize []uint `mapstructure:"additionaldisk_size"` - DiskName string `mapstructure:"vmdk_name"` - DiskSize uint `mapstructure:"disk_size"` - DiskTypeId string `mapstructure:"disk_type_id"` - FloppyFiles []string `mapstructure:"floppy_files"` - GuestOSType string `mapstructure:"guest_os_type"` - ISOChecksum string `mapstructure:"iso_checksum"` - ISOChecksumType string `mapstructure:"iso_checksum_type"` - ISOUrls []string `mapstructure:"iso_urls"` - Version string `mapstructure:"version"` - VMName string `mapstructure:"vm_name"` - BootCommand []string `mapstructure:"boot_command"` - SkipCompaction bool `mapstructure:"skip_compaction"` - VMXTemplatePath string `mapstructure:"vmx_template_path"` + AdditionalDiskSize []uint `mapstructure:"disk_additional_size"` + DiskName string `mapstructure:"vmdk_name"` + DiskSize uint `mapstructure:"disk_size"` + DiskTypeId string `mapstructure:"disk_type_id"` + FloppyFiles []string `mapstructure:"floppy_files"` + GuestOSType string `mapstructure:"guest_os_type"` + ISOChecksum string `mapstructure:"iso_checksum"` + ISOChecksumType string `mapstructure:"iso_checksum_type"` + ISOUrls []string `mapstructure:"iso_urls"` + Version string `mapstructure:"version"` + VMName string `mapstructure:"vm_name"` + BootCommand []string `mapstructure:"boot_command"` + SkipCompaction bool `mapstructure:"skip_compaction"` + VMXTemplatePath string `mapstructure:"vmx_template_path"` RemoteType string `mapstructure:"remote_type"` RemoteDatastore string `mapstructure:"remote_datastore"` diff --git a/website/source/docs/builders/vmware-iso.html.markdown b/website/source/docs/builders/vmware-iso.html.markdown index b021ca532..177bfc608 100644 --- a/website/source/docs/builders/vmware-iso.html.markdown +++ b/website/source/docs/builders/vmware-iso.html.markdown @@ -72,7 +72,7 @@ each category, the available options are alphabetized and described. ### Optional: -* `additionaldisk_size` (array of integers) - The size(s) of any additional +* `disk_additional_size` (array of integers) - The size(s) of any additional hard disks for the VM in megabytes. If this is not specified then the VM will only contain a primary hard disk. The builder uses expandable, not fixed-size virtual hard disks, so the actual file representing the disk will not use the From c9b413e3d1214f94fae52f91666e277a6c5fa85e Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Wed, 10 Jun 2015 15:18:05 -0500 Subject: [PATCH 32/85] builder/amazon-instance: Omit access, secrety key if using IAM Instance Profile --- builder/amazon/instance/builder.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index d164f91ee..91355b913 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -73,15 +73,25 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { } if b.config.BundleUploadCommand == "" { - b.config.BundleUploadCommand = "sudo -i -n ec2-upload-bundle " + - "-b {{.BucketName}} " + - "-m {{.ManifestPath}} " + - "-a {{.AccessKey}} " + - "-s {{.SecretKey}} " + - "-d {{.BundleDirectory}} " + - "--batch " + - "--region {{.Region}} " + - "--retry" + if b.config.IamInstanceProfile != "" { + b.config.BundleUploadCommand = "sudo -i -n ec2-upload-bundle " + + "-b {{.BucketName}} " + + "-m {{.ManifestPath}} " + + "-d {{.BundleDirectory}} " + + "--batch " + + "--region {{.Region}} " + + "--retry" + } else { + b.config.BundleUploadCommand = "sudo -i -n ec2-upload-bundle " + + "-b {{.BucketName}} " + + "-m {{.ManifestPath}} " + + "-a {{.AccessKey}} " + + "-s {{.SecretKey}} " + + "-d {{.BundleDirectory}} " + + "--batch " + + "--region {{.Region}} " + + "--retry" + } } if b.config.BundleVolCommand == "" { From d9c48e82fbc06d42d2b840eded67d344874ee53b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:02:06 -0700 Subject: [PATCH 33/85] builder/digitalocean: switch to new lib --- builder/digitalocean/api.go | 76 ---- builder/digitalocean/api_v1.go | 382 ---------------- builder/digitalocean/api_v2.go | 462 -------------------- builder/digitalocean/artifact.go | 9 +- builder/digitalocean/builder.go | 196 +-------- builder/digitalocean/builder_acc_test.go | 30 ++ builder/digitalocean/builder_test.go | 84 ---- builder/digitalocean/config.go | 140 ++++++ builder/digitalocean/step_create_droplet.go | 39 +- builder/digitalocean/step_create_ssh_key.go | 26 +- builder/digitalocean/step_droplet_info.go | 20 +- builder/digitalocean/step_power_off.go | 11 +- builder/digitalocean/step_shutdown.go | 9 +- builder/digitalocean/step_snapshot.go | 14 +- builder/digitalocean/token_source.go | 15 + builder/digitalocean/wait.go | 10 +- 16 files changed, 277 insertions(+), 1246 deletions(-) delete mode 100644 builder/digitalocean/api.go delete mode 100644 builder/digitalocean/api_v1.go delete mode 100644 builder/digitalocean/api_v2.go create mode 100644 builder/digitalocean/builder_acc_test.go create mode 100644 builder/digitalocean/config.go create mode 100644 builder/digitalocean/token_source.go diff --git a/builder/digitalocean/api.go b/builder/digitalocean/api.go deleted file mode 100644 index 87339ffc9..000000000 --- a/builder/digitalocean/api.go +++ /dev/null @@ -1,76 +0,0 @@ -// All of the methods used to communicate with the digital_ocean API -// are here. Their API is on a path to V2, so just plain JSON is used -// in place of a proper client library for now. - -package digitalocean - -type Region struct { - Slug string `json:"slug"` - Name string `json:"name"` - - // v1 only - Id uint `json:"id,omitempty"` - - // v2 only - Sizes []string `json:"sizes,omitempty"` - Available bool `json:"available,omitempty"` - Features []string `json:"features,omitempty"` -} - -type RegionsResp struct { - Regions []Region -} - -type Size struct { - Slug string `json:"slug"` - - // v1 only - Id uint `json:"id,omitempty"` - Name string `json:"name,omitempty"` - - // v2 only - Memory uint `json:"memory,omitempty"` - VCPUS uint `json:"vcpus,omitempty"` - Disk uint `json:"disk,omitempty"` - Transfer float64 `json:"transfer,omitempty"` - PriceMonthly float64 `json:"price_monthly,omitempty"` - PriceHourly float64 `json:"price_hourly,omitempty"` -} - -type SizesResp struct { - Sizes []Size -} - -type Image struct { - Id uint `json:"id"` - Name string `json:"name"` - Slug string `json:"slug"` - Distribution string `json:"distribution"` - - // v2 only - Public bool `json:"public,omitempty"` - ActionIds []string `json:"action_ids,omitempty"` - CreatedAt string `json:"created_at,omitempty"` -} - -type ImagesResp struct { - Images []Image -} - -type DigitalOceanClient interface { - CreateKey(string, string) (uint, error) - DestroyKey(uint) error - CreateDroplet(string, string, string, string, uint, bool) (uint, error) - DestroyDroplet(uint) error - PowerOffDroplet(uint) error - ShutdownDroplet(uint) error - CreateSnapshot(uint, string) error - Images() ([]Image, error) - DestroyImage(uint) error - DropletStatus(uint) (string, string, error) - Image(string) (Image, error) - Regions() ([]Region, error) - Region(string) (Region, error) - Sizes() ([]Size, error) - Size(string) (Size, error) -} diff --git a/builder/digitalocean/api_v1.go b/builder/digitalocean/api_v1.go deleted file mode 100644 index 23746d11f..000000000 --- a/builder/digitalocean/api_v1.go +++ /dev/null @@ -1,382 +0,0 @@ -// All of the methods used to communicate with the digital_ocean API -// are here. Their API is on a path to V2, so just plain JSON is used -// in place of a proper client library for now. - -package digitalocean - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "log" - "net/http" - "net/url" - "strconv" - "strings" - "time" - - "github.com/mitchellh/mapstructure" -) - -type DigitalOceanClientV1 struct { - // The http client for communicating - client *http.Client - - // Credentials - ClientID string - APIKey string - // The base URL of the API - APIURL string -} - -// Creates a new client for communicating with DO -func DigitalOceanClientNewV1(client string, key string, url string) *DigitalOceanClientV1 { - c := &DigitalOceanClientV1{ - client: &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - }, - }, - APIURL: url, - ClientID: client, - APIKey: key, - } - return c -} - -// Creates an SSH Key and returns it's id -func (d DigitalOceanClientV1) CreateKey(name string, pub string) (uint, error) { - params := url.Values{} - params.Set("name", name) - params.Set("ssh_pub_key", pub) - - body, err := NewRequestV1(d, "ssh_keys/new", params) - if err != nil { - return 0, err - } - - // Read the SSH key's ID we just created - key := body["ssh_key"].(map[string]interface{}) - keyId := key["id"].(float64) - return uint(keyId), nil -} - -// Destroys an SSH key -func (d DigitalOceanClientV1) DestroyKey(id uint) error { - path := fmt.Sprintf("ssh_keys/%v/destroy", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Creates a droplet and returns it's id -func (d DigitalOceanClientV1) CreateDroplet(name string, size string, image string, region string, keyId uint, privateNetworking bool) (uint, error) { - params := url.Values{} - params.Set("name", name) - - found_size, err := d.Size(size) - if err != nil { - return 0, fmt.Errorf("Invalid size or lookup failure: '%s': %s", size, err) - } - - found_image, err := d.Image(image) - if err != nil { - return 0, fmt.Errorf("Invalid image or lookup failure: '%s': %s", image, err) - } - - found_region, err := d.Region(region) - if err != nil { - return 0, fmt.Errorf("Invalid region or lookup failure: '%s': %s", region, err) - } - - params.Set("size_slug", found_size.Slug) - params.Set("image_slug", found_image.Slug) - params.Set("region_slug", found_region.Slug) - params.Set("ssh_key_ids", fmt.Sprintf("%v", keyId)) - params.Set("private_networking", fmt.Sprintf("%v", privateNetworking)) - - body, err := NewRequestV1(d, "droplets/new", params) - if err != nil { - return 0, err - } - - // Read the Droplets ID - droplet := body["droplet"].(map[string]interface{}) - dropletId := droplet["id"].(float64) - return uint(dropletId), err -} - -// Destroys a droplet -func (d DigitalOceanClientV1) DestroyDroplet(id uint) error { - path := fmt.Sprintf("droplets/%v/destroy", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Powers off a droplet -func (d DigitalOceanClientV1) PowerOffDroplet(id uint) error { - path := fmt.Sprintf("droplets/%v/power_off", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Shutsdown a droplet. This is a "soft" shutdown. -func (d DigitalOceanClientV1) ShutdownDroplet(id uint) error { - path := fmt.Sprintf("droplets/%v/shutdown", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Creates a snaphot of a droplet by it's ID -func (d DigitalOceanClientV1) CreateSnapshot(id uint, name string) error { - path := fmt.Sprintf("droplets/%v/snapshot", id) - - params := url.Values{} - params.Set("name", name) - - _, err := NewRequestV1(d, path, params) - - return err -} - -// Returns all available images. -func (d DigitalOceanClientV1) Images() ([]Image, error) { - resp, err := NewRequestV1(d, "images", url.Values{}) - if err != nil { - return nil, err - } - - var result ImagesResp - if err := mapstructure.Decode(resp, &result); err != nil { - return nil, err - } - - return result.Images, nil -} - -// Destroys an image by its ID. -func (d DigitalOceanClientV1) DestroyImage(id uint) error { - path := fmt.Sprintf("images/%d/destroy", id) - _, err := NewRequestV1(d, path, url.Values{}) - return err -} - -// Returns DO's string representation of status "off" "new" "active" etc. -func (d DigitalOceanClientV1) DropletStatus(id uint) (string, string, error) { - path := fmt.Sprintf("droplets/%v", id) - - body, err := NewRequestV1(d, path, url.Values{}) - if err != nil { - return "", "", err - } - - var ip string - - // Read the droplet's "status" - droplet := body["droplet"].(map[string]interface{}) - status := droplet["status"].(string) - - if droplet["ip_address"] != nil { - ip = droplet["ip_address"].(string) - } - - return ip, status, err -} - -// Sends an api request and returns a generic map[string]interface of -// the response. -func NewRequestV1(d DigitalOceanClientV1, path string, params url.Values) (map[string]interface{}, error) { - client := d.client - - // Add the authentication parameters - params.Set("client_id", d.ClientID) - params.Set("api_key", d.APIKey) - - url := fmt.Sprintf("%s/%s?%s", d.APIURL, path, params.Encode()) - - // Do some basic scrubbing so sensitive information doesn't appear in logs - scrubbedUrl := strings.Replace(url, d.ClientID, "CLIENT_ID", -1) - scrubbedUrl = strings.Replace(scrubbedUrl, d.APIKey, "API_KEY", -1) - log.Printf("sending new request to digitalocean: %s", scrubbedUrl) - - var lastErr error - for attempts := 1; attempts < 10; attempts++ { - resp, err := client.Get(url) - if err != nil { - return nil, err - } - - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return nil, err - } - - log.Printf("response from digitalocean: %s", body) - - var decodedResponse map[string]interface{} - err = json.Unmarshal(body, &decodedResponse) - if err != nil { - err = errors.New(fmt.Sprintf("Failed to decode JSON response (HTTP %v) from DigitalOcean: %s", - resp.StatusCode, body)) - return decodedResponse, err - } - - // Check for errors sent by digitalocean - status := decodedResponse["status"].(string) - if status == "OK" { - return decodedResponse, nil - } - - if status == "ERROR" { - statusRaw, ok := decodedResponse["error_message"] - if ok { - status = statusRaw.(string) - } else { - status = fmt.Sprintf( - "Unknown error. Full response body: %s", body) - } - } - - lastErr = errors.New(fmt.Sprintf("Received error from DigitalOcean (%d): %s", - resp.StatusCode, status)) - log.Println(lastErr) - if strings.Contains(status, "a pending event") { - // Retry, DigitalOcean sends these dumb "pending event" - // errors all the time. - time.Sleep(5 * time.Second) - continue - } - - // Some other kind of error. Just return. - return decodedResponse, lastErr - } - - return nil, lastErr -} - -func (d DigitalOceanClientV1) Image(slug_or_name_or_id string) (Image, error) { - images, err := d.Images() - if err != nil { - return Image{}, err - } - - for _, image := range images { - if strings.EqualFold(image.Slug, slug_or_name_or_id) { - return image, nil - } - } - - for _, image := range images { - if strings.EqualFold(image.Name, slug_or_name_or_id) { - return image, nil - } - } - - for _, image := range images { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if image.Id == uint(id) { - return image, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown image '%v'", slug_or_name_or_id)) - - return Image{}, err -} - -// Returns all available regions. -func (d DigitalOceanClientV1) Regions() ([]Region, error) { - resp, err := NewRequestV1(d, "regions", url.Values{}) - if err != nil { - return nil, err - } - - var result RegionsResp - if err := mapstructure.Decode(resp, &result); err != nil { - return nil, err - } - - return result.Regions, nil -} - -func (d DigitalOceanClientV1) Region(slug_or_name_or_id string) (Region, error) { - regions, err := d.Regions() - if err != nil { - return Region{}, err - } - - for _, region := range regions { - if strings.EqualFold(region.Slug, slug_or_name_or_id) { - return region, nil - } - } - - for _, region := range regions { - if strings.EqualFold(region.Name, slug_or_name_or_id) { - return region, nil - } - } - - for _, region := range regions { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if region.Id == uint(id) { - return region, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown region '%v'", slug_or_name_or_id)) - - return Region{}, err -} - -// Returns all available sizes. -func (d DigitalOceanClientV1) Sizes() ([]Size, error) { - resp, err := NewRequestV1(d, "sizes", url.Values{}) - if err != nil { - return nil, err - } - - var result SizesResp - if err := mapstructure.Decode(resp, &result); err != nil { - return nil, err - } - - return result.Sizes, nil -} - -func (d DigitalOceanClientV1) Size(slug_or_name_or_id string) (Size, error) { - sizes, err := d.Sizes() - if err != nil { - return Size{}, err - } - - for _, size := range sizes { - if strings.EqualFold(size.Slug, slug_or_name_or_id) { - return size, nil - } - } - - for _, size := range sizes { - if strings.EqualFold(size.Name, slug_or_name_or_id) { - return size, nil - } - } - - for _, size := range sizes { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if size.Id == uint(id) { - return size, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown size '%v'", slug_or_name_or_id)) - - return Size{}, err -} diff --git a/builder/digitalocean/api_v2.go b/builder/digitalocean/api_v2.go deleted file mode 100644 index 46454a9f8..000000000 --- a/builder/digitalocean/api_v2.go +++ /dev/null @@ -1,462 +0,0 @@ -// are here. Their API is on a path to V2, so just plain JSON is used -// in place of a proper client library for now. - -package digitalocean - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "log" - "net/http" - "strconv" - "strings" -) - -type DigitalOceanClientV2 struct { - // The http client for communicating - client *http.Client - - // Credentials - APIToken string - - // The base URL of the API - APIURL string -} - -// Creates a new client for communicating with DO -func DigitalOceanClientNewV2(token string, url string) *DigitalOceanClientV2 { - c := &DigitalOceanClientV2{ - client: &http.Client{ - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - }, - }, - APIURL: url, - APIToken: token, - } - return c -} - -// Creates an SSH Key and returns it's id -func (d DigitalOceanClientV2) CreateKey(name string, pub string) (uint, error) { - type KeyReq struct { - Name string `json:"name"` - PublicKey string `json:"public_key"` - } - type KeyRes struct { - SSHKey struct { - Id uint - Name string - Fingerprint string - PublicKey string `json:"public_key"` - } `json:"ssh_key"` - } - req := &KeyReq{Name: name, PublicKey: pub} - res := KeyRes{} - err := NewRequestV2(d, "v2/account/keys", "POST", req, &res) - if err != nil { - return 0, err - } - - return res.SSHKey.Id, err -} - -// Destroys an SSH key -func (d DigitalOceanClientV2) DestroyKey(id uint) error { - path := fmt.Sprintf("v2/account/keys/%v", id) - return NewRequestV2(d, path, "DELETE", nil, nil) -} - -// Creates a droplet and returns it's id -func (d DigitalOceanClientV2) CreateDroplet(name string, size string, image string, region string, keyId uint, privateNetworking bool) (uint, error) { - type DropletReq struct { - Name string `json:"name"` - Region string `json:"region"` - Size string `json:"size"` - Image string `json:"image"` - SSHKeys []string `json:"ssh_keys,omitempty"` - Backups bool `json:"backups,omitempty"` - IPv6 bool `json:"ipv6,omitempty"` - PrivateNetworking bool `json:"private_networking,omitempty"` - } - type DropletRes struct { - Droplet struct { - Id uint - Name string - Memory uint - VCPUS uint `json:"vcpus"` - Disk uint - Region Region - Image Image - Size Size - Locked bool - CreateAt string `json:"created_at"` - Status string - Networks struct { - V4 []struct { - IPAddr string `json:"ip_address"` - Netmask string - Gateway string - Type string - } `json:"v4,omitempty"` - V6 []struct { - IPAddr string `json:"ip_address"` - CIDR uint `json:"cidr"` - Gateway string - Type string - } `json:"v6,omitempty"` - } - Kernel struct { - Id uint - Name string - Version string - } - BackupIds []uint - SnapshotIds []uint - ActionIds []uint - Features []string `json:"features,omitempty"` - } - } - req := &DropletReq{Name: name} - res := DropletRes{} - - found_size, err := d.Size(size) - if err != nil { - return 0, fmt.Errorf("Invalid size or lookup failure: '%s': %s", size, err) - } - - found_image, err := d.Image(image) - if err != nil { - return 0, fmt.Errorf("Invalid image or lookup failure: '%s': %s", image, err) - } - - found_region, err := d.Region(region) - if err != nil { - return 0, fmt.Errorf("Invalid region or lookup failure: '%s': %s", region, err) - } - - if found_image.Slug == "" { - req.Image = strconv.Itoa(int(found_image.Id)) - } else { - req.Image = found_image.Slug - } - - req.Size = found_size.Slug - req.Region = found_region.Slug - req.SSHKeys = []string{fmt.Sprintf("%v", keyId)} - req.PrivateNetworking = privateNetworking - - err = NewRequestV2(d, "v2/droplets", "POST", req, &res) - if err != nil { - return 0, err - } - - return res.Droplet.Id, err -} - -// Destroys a droplet -func (d DigitalOceanClientV2) DestroyDroplet(id uint) error { - path := fmt.Sprintf("v2/droplets/%v", id) - return NewRequestV2(d, path, "DELETE", nil, nil) -} - -// Powers off a droplet -func (d DigitalOceanClientV2) PowerOffDroplet(id uint) error { - type ActionReq struct { - Type string `json:"type"` - } - type ActionRes struct { - } - req := &ActionReq{Type: "power_off"} - path := fmt.Sprintf("v2/droplets/%v/actions", id) - return NewRequestV2(d, path, "POST", req, nil) -} - -// Shutsdown a droplet. This is a "soft" shutdown. -func (d DigitalOceanClientV2) ShutdownDroplet(id uint) error { - type ActionReq struct { - Type string `json:"type"` - } - type ActionRes struct { - } - req := &ActionReq{Type: "shutdown"} - - path := fmt.Sprintf("v2/droplets/%v/actions", id) - return NewRequestV2(d, path, "POST", req, nil) -} - -// Creates a snaphot of a droplet by it's ID -func (d DigitalOceanClientV2) CreateSnapshot(id uint, name string) error { - type ActionReq struct { - Type string `json:"type"` - Name string `json:"name"` - } - type ActionRes struct { - } - req := &ActionReq{Type: "snapshot", Name: name} - path := fmt.Sprintf("v2/droplets/%v/actions", id) - return NewRequestV2(d, path, "POST", req, nil) -} - -// Returns all available images. -func (d DigitalOceanClientV2) Images() ([]Image, error) { - res := ImagesResp{} - - err := NewRequestV2(d, "v2/images?per_page=200", "GET", nil, &res) - if err != nil { - return nil, err - } - - return res.Images, nil -} - -// Destroys an image by its ID. -func (d DigitalOceanClientV2) DestroyImage(id uint) error { - path := fmt.Sprintf("v2/images/%d", id) - return NewRequestV2(d, path, "DELETE", nil, nil) -} - -// Returns DO's string representation of status "off" "new" "active" etc. -func (d DigitalOceanClientV2) DropletStatus(id uint) (string, string, error) { - path := fmt.Sprintf("v2/droplets/%v", id) - type DropletRes struct { - Droplet struct { - Id uint - Name string - Memory uint - VCPUS uint `json:"vcpus"` - Disk uint - Region Region - Image Image - Size Size - Locked bool - CreateAt string `json:"created_at"` - Status string - Networks struct { - V4 []struct { - IPAddr string `json:"ip_address"` - Netmask string - Gateway string - Type string - } `json:"v4,omitempty"` - V6 []struct { - IPAddr string `json:"ip_address"` - CIDR uint `json:"cidr"` - Gateway string - Type string - } `json:"v6,omitempty"` - } - Kernel struct { - Id uint - Name string - Version string - } - BackupIds []uint - SnapshotIds []uint - ActionIds []uint - Features []string `json:"features,omitempty"` - } - } - res := DropletRes{} - err := NewRequestV2(d, path, "GET", nil, &res) - if err != nil { - return "", "", err - } - var ip string - - for _, n := range res.Droplet.Networks.V4 { - if n.Type == "public" { - ip = n.IPAddr - } - } - - return ip, res.Droplet.Status, err -} - -// Sends an api request and returns a generic map[string]interface of -// the response. -func NewRequestV2(d DigitalOceanClientV2, path string, method string, req interface{}, res interface{}) error { - var err error - var request *http.Request - - client := d.client - - buf := new(bytes.Buffer) - // Add the authentication parameters - url := fmt.Sprintf("%s/%s", d.APIURL, path) - if req != nil { - enc := json.NewEncoder(buf) - enc.Encode(req) - defer buf.Reset() - request, err = http.NewRequest(method, url, buf) - request.Header.Add("Content-Type", "application/json") - } else { - request, err = http.NewRequest(method, url, nil) - } - if err != nil { - return err - } - - // Add the authentication parameters - request.Header.Add("Authorization", "Bearer "+d.APIToken) - if buf != nil { - log.Printf("sending new request to digitalocean: %s buffer: %s", url, buf) - } else { - log.Printf("sending new request to digitalocean: %s", url) - } - resp, err := client.Do(request) - if err != nil { - return err - } - - if method == "DELETE" && resp.StatusCode == 204 { - if resp.Body != nil { - resp.Body.Close() - } - return nil - } - - if resp.Body == nil { - return errors.New("Request returned empty body") - } - - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - return err - } - - log.Printf("response from digitalocean: %s", body) - - err = json.Unmarshal(body, &res) - if err != nil { - return errors.New(fmt.Sprintf("Failed to decode JSON response %s (HTTP %v) from DigitalOcean: %s", err.Error(), - resp.StatusCode, body)) - } - switch resp.StatusCode { - case 403, 401, 429, 422, 404, 503, 500: - return errors.New(fmt.Sprintf("digitalocean request error: %+v", res)) - } - return nil -} - -func (d DigitalOceanClientV2) Image(slug_or_name_or_id string) (Image, error) { - images, err := d.Images() - if err != nil { - return Image{}, err - } - - for _, image := range images { - if strings.EqualFold(image.Slug, slug_or_name_or_id) { - return image, nil - } - } - - for _, image := range images { - if strings.EqualFold(image.Name, slug_or_name_or_id) { - return image, nil - } - } - - for _, image := range images { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if image.Id == uint(id) { - return image, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown image '%v'", slug_or_name_or_id)) - - return Image{}, err -} - -// Returns all available regions. -func (d DigitalOceanClientV2) Regions() ([]Region, error) { - res := RegionsResp{} - err := NewRequestV2(d, "v2/regions?per_page=200", "GET", nil, &res) - if err != nil { - return nil, err - } - - return res.Regions, nil -} - -func (d DigitalOceanClientV2) Region(slug_or_name_or_id string) (Region, error) { - regions, err := d.Regions() - if err != nil { - return Region{}, err - } - - for _, region := range regions { - if strings.EqualFold(region.Slug, slug_or_name_or_id) { - return region, nil - } - } - - for _, region := range regions { - if strings.EqualFold(region.Name, slug_or_name_or_id) { - return region, nil - } - } - - for _, region := range regions { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if region.Id == uint(id) { - return region, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown region '%v'", slug_or_name_or_id)) - - return Region{}, err -} - -// Returns all available sizes. -func (d DigitalOceanClientV2) Sizes() ([]Size, error) { - res := SizesResp{} - err := NewRequestV2(d, "v2/sizes?per_page=200", "GET", nil, &res) - if err != nil { - return nil, err - } - - return res.Sizes, nil -} - -func (d DigitalOceanClientV2) Size(slug_or_name_or_id string) (Size, error) { - sizes, err := d.Sizes() - if err != nil { - return Size{}, err - } - - for _, size := range sizes { - if strings.EqualFold(size.Slug, slug_or_name_or_id) { - return size, nil - } - } - - for _, size := range sizes { - if strings.EqualFold(size.Name, slug_or_name_or_id) { - return size, nil - } - } - - for _, size := range sizes { - id, err := strconv.Atoi(slug_or_name_or_id) - if err == nil { - if size.Id == uint(id) { - return size, nil - } - } - } - - err = errors.New(fmt.Sprintf("Unknown size '%v'", slug_or_name_or_id)) - - return Size{}, err -} diff --git a/builder/digitalocean/artifact.go b/builder/digitalocean/artifact.go index d1d878193..3b6a05e53 100644 --- a/builder/digitalocean/artifact.go +++ b/builder/digitalocean/artifact.go @@ -4,6 +4,8 @@ import ( "fmt" "log" "strconv" + + "github.com/digitalocean/godo" ) type Artifact struct { @@ -11,13 +13,13 @@ type Artifact struct { snapshotName string // The ID of the image - snapshotId uint + snapshotId int // The name of the region regionName string // The client for making API calls - client DigitalOceanClient + client *godo.Client } func (*Artifact) BuilderId() string { @@ -43,5 +45,6 @@ func (a *Artifact) State(name string) interface{} { func (a *Artifact) Destroy() error { log.Printf("Destroying image: %d (%s)", a.snapshotId, a.snapshotName) - return a.client.DestroyImage(a.snapshotId) + _, err := a.client.Images.Delete(a.snapshotId) + return err } diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index 3292bea10..3ba7074a2 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -4,18 +4,14 @@ package digitalocean import ( - "errors" - "fmt" "log" - "os" "time" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common" - "github.com/mitchellh/packer/common/uuid" - "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" - "github.com/mitchellh/packer/template/interpolate" + "golang.org/x/oauth2" ) // see https://api.digitalocean.com/images/?client_id=[client_id]&api_key=[api_key] @@ -33,179 +29,25 @@ const DefaultSize = "512mb" // The unique id for the builder const BuilderId = "pearkes.digitalocean" -// Configuration tells the builder the credentials -// to use while communicating with DO and describes the image -// you are creating -type Config struct { - common.PackerConfig `mapstructure:",squash"` - - ClientID string `mapstructure:"client_id"` - APIKey string `mapstructure:"api_key"` - APIURL string `mapstructure:"api_url"` - APIToken string `mapstructure:"api_token"` - RegionID uint `mapstructure:"region_id"` - SizeID uint `mapstructure:"size_id"` - ImageID uint `mapstructure:"image_id"` - - Region string `mapstructure:"region"` - Size string `mapstructure:"size"` - Image string `mapstructure:"image"` - - PrivateNetworking bool `mapstructure:"private_networking"` - SnapshotName string `mapstructure:"snapshot_name"` - DropletName string `mapstructure:"droplet_name"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort uint `mapstructure:"ssh_port"` - - RawSSHTimeout string `mapstructure:"ssh_timeout"` - RawStateTimeout string `mapstructure:"state_timeout"` - - // These are unexported since they're set by other fields - // being set. - sshTimeout time.Duration - stateTimeout time.Duration - - ctx *interpolate.Context -} - type Builder struct { config Config runner multistep.Runner } func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { - err := config.Decode(&b.config, &config.DecodeOpts{ - Interpolate: true, - }, raws...) - if err != nil { - return nil, err + c, warnings, errs := NewConfig(raws...) + if errs != nil { + return warnings, errs } + b.config = *c - // Optional configuration with defaults - if b.config.APIKey == "" { - // Default to environment variable for api_key, if it exists - b.config.APIKey = os.Getenv("DIGITALOCEAN_API_KEY") - } - - if b.config.ClientID == "" { - // Default to environment variable for client_id, if it exists - b.config.ClientID = os.Getenv("DIGITALOCEAN_CLIENT_ID") - } - - if b.config.APIURL == "" { - // Default to environment variable for api_url, if it exists - b.config.APIURL = os.Getenv("DIGITALOCEAN_API_URL") - } - - if b.config.APIToken == "" { - // Default to environment variable for api_token, if it exists - b.config.APIToken = os.Getenv("DIGITALOCEAN_API_TOKEN") - } - - if b.config.Region == "" { - if b.config.RegionID != 0 { - b.config.Region = fmt.Sprintf("%v", b.config.RegionID) - } else { - b.config.Region = DefaultRegion - } - } - - if b.config.Size == "" { - if b.config.SizeID != 0 { - b.config.Size = fmt.Sprintf("%v", b.config.SizeID) - } else { - b.config.Size = DefaultSize - } - } - - if b.config.Image == "" { - if b.config.ImageID != 0 { - b.config.Image = fmt.Sprintf("%v", b.config.ImageID) - } else { - b.config.Image = DefaultImage - } - } - - if b.config.SnapshotName == "" { - // Default to packer-{{ unix timestamp (utc) }} - b.config.SnapshotName = "packer-{{timestamp}}" - } - - if b.config.DropletName == "" { - // Default to packer-[time-ordered-uuid] - b.config.DropletName = fmt.Sprintf("packer-%s", uuid.TimeOrderedUUID()) - } - - if b.config.SSHUsername == "" { - // Default to "root". You can override this if your - // SourceImage has a different user account then the DO default - b.config.SSHUsername = "root" - } - - if b.config.SSHPort == 0 { - // Default to port 22 per DO default - b.config.SSHPort = 22 - } - - if b.config.RawSSHTimeout == "" { - // Default to 1 minute timeouts - b.config.RawSSHTimeout = "1m" - } - - if b.config.RawStateTimeout == "" { - // Default to 6 minute timeouts waiting for - // desired state. i.e waiting for droplet to become active - b.config.RawStateTimeout = "6m" - } - - var errs *packer.MultiError - if b.config.APIToken == "" { - // Required configurations that will display errors if not set - if b.config.ClientID == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("a client_id for v1 auth or api_token for v2 auth must be specified")) - } - - if b.config.APIKey == "" { - errs = packer.MultiErrorAppend( - errs, errors.New("a api_key for v1 auth or api_token for v2 auth must be specified")) - } - } - - if b.config.APIURL == "" { - b.config.APIURL = "https://api.digitalocean.com" - } - - sshTimeout, err := time.ParseDuration(b.config.RawSSHTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - b.config.sshTimeout = sshTimeout - - stateTimeout, err := time.ParseDuration(b.config.RawStateTimeout) - if err != nil { - errs = packer.MultiErrorAppend( - errs, fmt.Errorf("Failed parsing state_timeout: %s", err)) - } - b.config.stateTimeout = stateTimeout - - if errs != nil && len(errs.Errors) > 0 { - return nil, errs - } - - common.ScrubConfig(b.config, b.config.ClientID, b.config.APIKey) return nil, nil } func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { - var client DigitalOceanClient - // Initialize the DO API client - if b.config.APIToken == "" { - client = DigitalOceanClientNewV1(b.config.ClientID, b.config.APIKey, b.config.APIURL) - } else { - client = DigitalOceanClientNewV2(b.config.APIToken, b.config.APIURL) - } + client := godo.NewClient(oauth2.NewClient(oauth2.NoContext, &apiTokenSource{ + AccessToken: b.config.APIToken, + })) // Set up the state state := new(multistep.BasicStateBag) @@ -252,26 +94,10 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe return nil, nil } - sregion := state.Get("region") - - var region string - - if sregion != nil { - region = sregion.(string) - } else { - region = fmt.Sprintf("%v", state.Get("region_id").(uint)) - } - - found_region, err := client.Region(region) - - if err != nil { - return nil, err - } - artifact := &Artifact{ snapshotName: state.Get("snapshot_name").(string), - snapshotId: state.Get("snapshot_image_id").(uint), - regionName: found_region.Name, + snapshotId: state.Get("snapshot_image_id").(int), + regionName: state.Get("region").(string), client: client, } diff --git a/builder/digitalocean/builder_acc_test.go b/builder/digitalocean/builder_acc_test.go new file mode 100644 index 000000000..20e56b924 --- /dev/null +++ b/builder/digitalocean/builder_acc_test.go @@ -0,0 +1,30 @@ +package digitalocean + +import ( + "os" + "testing" + + builderT "github.com/mitchellh/packer/helper/builder/testing" +) + +func TestBuilderAcc_basic(t *testing.T) { + builderT.Test(t, builderT.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Builder: &Builder{}, + Template: testBuilderAccBasic, + }) +} + +func testAccPreCheck(t *testing.T) { + if v := os.Getenv("DIGITALOCEAN_API_TOKEN"); v == "" { + t.Fatal("DIGITALOCEAN_API_TOKEN must be set for acceptance tests") + } +} + +const testBuilderAccBasic = ` +{ + "builders": [{ + "type": "test" + }] +} +` diff --git a/builder/digitalocean/builder_test.go b/builder/digitalocean/builder_test.go index bd3bb1d21..8985aae2f 100644 --- a/builder/digitalocean/builder_test.go +++ b/builder/digitalocean/builder_test.go @@ -43,90 +43,6 @@ func TestBuilder_Prepare_BadType(t *testing.T) { } } -func TestBuilderPrepare_APIKey(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["api_key"] = "foo" - warnings, err := b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.APIKey != "foo" { - t.Errorf("access key invalid: %s", b.config.APIKey) - } - - // Test bad - delete(config, "api_key") - b = Builder{} - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err == nil { - t.Fatal("should have error") - } - - // Test env variable - delete(config, "api_key") - os.Setenv("DIGITALOCEAN_API_KEY", "foo") - defer os.Setenv("DIGITALOCEAN_API_KEY", "") - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } -} - -func TestBuilderPrepare_ClientID(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["client_id"] = "foo" - warnings, err := b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.ClientID != "foo" { - t.Errorf("invalid: %s", b.config.ClientID) - } - - // Test bad - delete(config, "client_id") - b = Builder{} - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err == nil { - t.Fatal("should have error") - } - - // Test env variable - delete(config, "client_id") - os.Setenv("DIGITALOCEAN_CLIENT_ID", "foo") - defer os.Setenv("DIGITALOCEAN_CLIENT_ID", "") - warnings, err = b.Prepare(config) - if len(warnings) > 0 { - t.Fatalf("bad: %#v", warnings) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } -} - func TestBuilderPrepare_InvalidKey(t *testing.T) { var b Builder config := testConfig() diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go new file mode 100644 index 000000000..0be49f08e --- /dev/null +++ b/builder/digitalocean/config.go @@ -0,0 +1,140 @@ +package digitalocean + +import ( + "errors" + "fmt" + "os" + "time" + + "github.com/mitchellh/mapstructure" + "github.com/mitchellh/packer/common" + "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/helper/config" + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/template/interpolate" + //"github.com/digitalocean/godo" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + APIToken string `mapstructure:"api_token"` + + // OLD STUFF + + Region string `mapstructure:"region"` + Size string `mapstructure:"size"` + Image string `mapstructure:"image"` + + PrivateNetworking bool `mapstructure:"private_networking"` + SnapshotName string `mapstructure:"snapshot_name"` + DropletName string `mapstructure:"droplet_name"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort uint `mapstructure:"ssh_port"` + + RawSSHTimeout string `mapstructure:"ssh_timeout"` + RawStateTimeout string `mapstructure:"state_timeout"` + + // These are unexported since they're set by other fields + // being set. + sshTimeout time.Duration + stateTimeout time.Duration + + ctx *interpolate.Context +} + +func NewConfig(raws ...interface{}) (*Config, []string, error) { + var c Config + + var md mapstructure.Metadata + err := config.Decode(&c, &config.DecodeOpts{ + Metadata: &md, + Interpolate: true, + InterpolateFilter: &interpolate.RenderFilter{ + Exclude: []string{ + "run_command", + }, + }, + }, raws...) + if err != nil { + return nil, nil, err + } + + // Defaults + if c.APIToken == "" { + // Default to environment variable for api_token, if it exists + c.APIToken = os.Getenv("DIGITALOCEAN_API_TOKEN") + } + + if c.Region == "" { + c.Region = DefaultRegion + } + + if c.Size == "" { + c.Size = DefaultSize + } + + if c.Image == "" { + c.Image = DefaultImage + } + + if c.SnapshotName == "" { + // Default to packer-{{ unix timestamp (utc) }} + c.SnapshotName = "packer-{{timestamp}}" + } + + if c.DropletName == "" { + // Default to packer-[time-ordered-uuid] + c.DropletName = fmt.Sprintf("packer-%s", uuid.TimeOrderedUUID()) + } + + if c.SSHUsername == "" { + // Default to "root". You can override this if your + // SourceImage has a different user account then the DO default + c.SSHUsername = "root" + } + + if c.SSHPort == 0 { + // Default to port 22 per DO default + c.SSHPort = 22 + } + + if c.RawSSHTimeout == "" { + // Default to 1 minute timeouts + c.RawSSHTimeout = "1m" + } + + if c.RawStateTimeout == "" { + // Default to 6 minute timeouts waiting for + // desired state. i.e waiting for droplet to become active + c.RawStateTimeout = "6m" + } + + var errs *packer.MultiError + if c.APIToken == "" { + // Required configurations that will display errors if not set + errs = packer.MultiErrorAppend( + errs, errors.New("api_token for auth must be specified")) + } + + sshTimeout, err := time.ParseDuration(c.RawSSHTimeout) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) + } + c.sshTimeout = sshTimeout + + stateTimeout, err := time.ParseDuration(c.RawStateTimeout) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Failed parsing state_timeout: %s", err)) + } + c.stateTimeout = stateTimeout + + if errs != nil && len(errs.Errors) > 0 { + return nil, nil, errs + } + + common.ScrubConfig(c, c.APIToken) + return &c, nil, nil +} diff --git a/builder/digitalocean/step_create_droplet.go b/builder/digitalocean/step_create_droplet.go index afb3e5814..40ac8f0e9 100644 --- a/builder/digitalocean/step_create_droplet.go +++ b/builder/digitalocean/step_create_droplet.go @@ -3,25 +3,35 @@ package digitalocean import ( "fmt" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) type stepCreateDroplet struct { - dropletId uint + dropletId int } func (s *stepCreateDroplet) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) c := state.Get("config").(Config) - sshKeyId := state.Get("ssh_key_id").(uint) - - ui.Say("Creating droplet...") + sshKeyId := state.Get("ssh_key_id").(int) // Create the droplet based on configuration - dropletId, err := client.CreateDroplet(c.DropletName, c.Size, c.Image, c.Region, sshKeyId, c.PrivateNetworking) - + ui.Say("Creating droplet...") + droplet, _, err := client.Droplets.Create(&godo.DropletCreateRequest{ + Name: c.DropletName, + Region: c.Region, + Size: c.Size, + Image: godo.DropletCreateImage{ + Slug: c.Image, + }, + SSHKeys: []godo.DropletCreateSSHKey{ + godo.DropletCreateSSHKey{ID: int(sshKeyId)}, + }, + PrivateNetworking: c.PrivateNetworking, + }) if err != nil { err := fmt.Errorf("Error creating droplet: %s", err) state.Put("error", err) @@ -30,10 +40,10 @@ func (s *stepCreateDroplet) Run(state multistep.StateBag) multistep.StepAction { } // We use this in cleanup - s.dropletId = dropletId + s.dropletId = droplet.ID // Store the droplet id for later - state.Put("droplet_id", dropletId) + state.Put("droplet_id", droplet.ID) return multistep.ActionContinue } @@ -44,19 +54,14 @@ func (s *stepCreateDroplet) Cleanup(state multistep.StateBag) { return } - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) - c := state.Get("config").(Config) // Destroy the droplet we just created ui.Say("Destroying droplet...") - - err := client.DestroyDroplet(s.dropletId) + _, err := client.Droplets.Delete(s.dropletId) if err != nil { - curlstr := fmt.Sprintf("curl '%v/droplets/%v/destroy?client_id=%v&api_key=%v'", - c.APIURL, s.dropletId, c.ClientID, c.APIKey) - ui.Error(fmt.Sprintf( - "Error destroying droplet. Please destroy it manually: %v", curlstr)) + "Error destroying droplet. Please destroy it manually: %s", err)) } } diff --git a/builder/digitalocean/step_create_ssh_key.go b/builder/digitalocean/step_create_ssh_key.go index db1ad9c16..fa0940c23 100644 --- a/builder/digitalocean/step_create_ssh_key.go +++ b/builder/digitalocean/step_create_ssh_key.go @@ -9,17 +9,18 @@ import ( "log" "code.google.com/p/gosshold/ssh" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/common/uuid" "github.com/mitchellh/packer/packer" ) type stepCreateSSHKey struct { - keyId uint + keyId int } func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) ui.Say("Creating temporary ssh key for droplet...") @@ -46,7 +47,10 @@ func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { name := fmt.Sprintf("packer-%s", uuid.TimeOrderedUUID()) // Create the key! - keyId, err := client.CreateKey(name, pub_sshformat) + key, _, err := client.Keys.Create(&godo.KeyCreateRequest{ + Name: name, + PublicKey: pub_sshformat, + }) if err != nil { err := fmt.Errorf("Error creating temporary SSH key: %s", err) state.Put("error", err) @@ -55,12 +59,12 @@ func (s *stepCreateSSHKey) Run(state multistep.StateBag) multistep.StepAction { } // We use this to check cleanup - s.keyId = keyId + s.keyId = key.ID log.Printf("temporary ssh key name: %s", name) // Remember some state for the future - state.Put("ssh_key_id", keyId) + state.Put("ssh_key_id", key.ID) return multistep.ActionContinue } @@ -71,18 +75,14 @@ func (s *stepCreateSSHKey) Cleanup(state multistep.StateBag) { return } - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) - c := state.Get("config").(Config) ui.Say("Deleting temporary ssh key...") - err := client.DestroyKey(s.keyId) - - curlstr := fmt.Sprintf("curl -H 'Authorization: Bearer #TOKEN#' -X DELETE '%v/v2/account/keys/%v'", c.APIURL, s.keyId) - + _, err := client.Keys.DeleteByID(s.keyId) if err != nil { - log.Printf("Error cleaning up ssh key: %v", err.Error()) + log.Printf("Error cleaning up ssh key: %s", err) ui.Error(fmt.Sprintf( - "Error cleaning up ssh key. Please delete the key manually: %v", curlstr)) + "Error cleaning up ssh key. Please delete the key manually: %s", err)) } } diff --git a/builder/digitalocean/step_droplet_info.go b/builder/digitalocean/step_droplet_info.go index 8e9b69927..5fbcb7141 100644 --- a/builder/digitalocean/step_droplet_info.go +++ b/builder/digitalocean/step_droplet_info.go @@ -3,6 +3,7 @@ package digitalocean import ( "fmt" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) @@ -10,10 +11,10 @@ import ( type stepDropletInfo struct{} func (s *stepDropletInfo) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) c := state.Get("config").(Config) - dropletId := state.Get("droplet_id").(uint) + dropletId := state.Get("droplet_id").(int) ui.Say("Waiting for droplet to become active...") @@ -26,16 +27,25 @@ func (s *stepDropletInfo) Run(state multistep.StateBag) multistep.StepAction { } // Set the IP on the state for later - ip, _, err := client.DropletStatus(dropletId) + droplet, _, err := client.Droplets.Get(dropletId) if err != nil { - err := fmt.Errorf("Error retrieving droplet ID: %s", err) + err := fmt.Errorf("Error retrieving droplet: %s", err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - state.Put("droplet_ip", ip) + // Verify we have an IPv4 address + invalid := droplet.Networks == nil || + len(droplet.Networks.V4) == 0 + if invalid { + err := fmt.Errorf("IPv4 address not found for droplet!") + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + state.Put("droplet_ip", droplet.Networks.V4[0].IPAddress) return multistep.ActionContinue } diff --git a/builder/digitalocean/step_power_off.go b/builder/digitalocean/step_power_off.go index d6ef49a22..3d547e8c2 100644 --- a/builder/digitalocean/step_power_off.go +++ b/builder/digitalocean/step_power_off.go @@ -4,6 +4,7 @@ import ( "fmt" "log" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) @@ -11,12 +12,12 @@ import ( type stepPowerOff struct{} func (s *stepPowerOff) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) c := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) - dropletId := state.Get("droplet_id").(uint) + dropletId := state.Get("droplet_id").(int) - _, status, err := client.DropletStatus(dropletId) + droplet, _, err := client.Droplets.Get(dropletId) if err != nil { err := fmt.Errorf("Error checking droplet state: %s", err) state.Put("error", err) @@ -24,14 +25,14 @@ func (s *stepPowerOff) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - if status == "off" { + if droplet.Status == "off" { // Droplet is already off, don't do anything return multistep.ActionContinue } // Pull the plug on the Droplet ui.Say("Forcefully shutting down Droplet...") - err = client.PowerOffDroplet(dropletId) + _, _, err = client.DropletActions.PowerOff(dropletId) if err != nil { err := fmt.Errorf("Error powering off droplet: %s", err) state.Put("error", err) diff --git a/builder/digitalocean/step_shutdown.go b/builder/digitalocean/step_shutdown.go index 06a2ae9f5..602f3e690 100644 --- a/builder/digitalocean/step_shutdown.go +++ b/builder/digitalocean/step_shutdown.go @@ -5,6 +5,7 @@ import ( "log" "time" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) @@ -12,16 +13,16 @@ import ( type stepShutdown struct{} func (s *stepShutdown) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) - dropletId := state.Get("droplet_id").(uint) + dropletId := state.Get("droplet_id").(int) // Gracefully power off the droplet. We have to retry this a number // of times because sometimes it says it completed when it actually // did absolutely nothing (*ALAKAZAM!* magic!). We give up after // a pretty arbitrary amount of time. ui.Say("Gracefully shutting down droplet...") - err := client.ShutdownDroplet(dropletId) + _, _, err := client.DropletActions.Shutdown(dropletId) if err != nil { // If we get an error the first time, actually report it err := fmt.Errorf("Error shutting down droplet: %s", err) @@ -48,7 +49,7 @@ func (s *stepShutdown) Run(state multistep.StateBag) multistep.StepAction { for attempts := 2; attempts > 0; attempts++ { log.Printf("ShutdownDroplet attempt #%d...", attempts) - err := client.ShutdownDroplet(dropletId) + _, _, err := client.DropletActions.Shutdown(dropletId) if err != nil { log.Printf("Shutdown retry error: %s", err) } diff --git a/builder/digitalocean/step_snapshot.go b/builder/digitalocean/step_snapshot.go index 1903c1a34..7ff384924 100644 --- a/builder/digitalocean/step_snapshot.go +++ b/builder/digitalocean/step_snapshot.go @@ -5,6 +5,7 @@ import ( "fmt" "log" + "github.com/digitalocean/godo" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" ) @@ -12,13 +13,13 @@ import ( type stepSnapshot struct{} func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { - client := state.Get("client").(DigitalOceanClient) + client := state.Get("client").(*godo.Client) ui := state.Get("ui").(packer.Ui) c := state.Get("config").(Config) - dropletId := state.Get("droplet_id").(uint) + dropletId := state.Get("droplet_id").(int) ui.Say(fmt.Sprintf("Creating snapshot: %v", c.SnapshotName)) - err := client.CreateSnapshot(dropletId, c.SnapshotName) + _, _, err := client.DropletActions.Snapshot(dropletId, c.SnapshotName) if err != nil { err := fmt.Errorf("Error creating snapshot: %s", err) state.Put("error", err) @@ -36,7 +37,7 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { } log.Printf("Looking up snapshot ID for snapshot: %s", c.SnapshotName) - images, err := client.Images() + images, _, err := client.Images.List(nil) if err != nil { err := fmt.Errorf("Error looking up snapshot ID: %s", err) state.Put("error", err) @@ -44,10 +45,10 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - var imageId uint + var imageId int for _, image := range images { if image.Name == c.SnapshotName { - imageId = image.Id + imageId = image.ID break } } @@ -60,7 +61,6 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { } log.Printf("Snapshot image ID: %d", imageId) - state.Put("snapshot_image_id", imageId) state.Put("snapshot_name", c.SnapshotName) state.Put("region", c.Region) diff --git a/builder/digitalocean/token_source.go b/builder/digitalocean/token_source.go new file mode 100644 index 000000000..eab5a084b --- /dev/null +++ b/builder/digitalocean/token_source.go @@ -0,0 +1,15 @@ +package digitalocean + +import ( + "golang.org/x/oauth2" +) + +type apiTokenSource struct { + AccessToken string +} + +func (t *apiTokenSource) Token() (*oauth2.Token, error) { + return &oauth2.Token{ + AccessToken: t.AccessToken, + }, nil +} diff --git a/builder/digitalocean/wait.go b/builder/digitalocean/wait.go index e5b1dee90..3d299d433 100644 --- a/builder/digitalocean/wait.go +++ b/builder/digitalocean/wait.go @@ -4,11 +4,15 @@ import ( "fmt" "log" "time" + + "github.com/digitalocean/godo" ) // waitForState simply blocks until the droplet is in // a state we expect, while eventually timing out. -func waitForDropletState(desiredState string, dropletId uint, client DigitalOceanClient, timeout time.Duration) error { +func waitForDropletState( + desiredState string, dropletId int, + client *godo.Client, timeout time.Duration) error { done := make(chan struct{}) defer close(done) @@ -19,13 +23,13 @@ func waitForDropletState(desiredState string, dropletId uint, client DigitalOcea attempts += 1 log.Printf("Checking droplet status... (attempt: %d)", attempts) - _, status, err := client.DropletStatus(dropletId) + droplet, _, err := client.Droplets.Get(dropletId) if err != nil { result <- err return } - if status == desiredState { + if droplet.Status == desiredState { result <- nil return } From 486c7e4ae61237a82fb0ec1be57d9adde197deb8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:07:24 -0700 Subject: [PATCH 34/85] builder/digitalocean: remove unused things --- builder/digitalocean/config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 0be49f08e..5defe89db 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -12,7 +12,6 @@ import ( "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" - //"github.com/digitalocean/godo" ) type Config struct { @@ -20,8 +19,6 @@ type Config struct { APIToken string `mapstructure:"api_token"` - // OLD STUFF - Region string `mapstructure:"region"` Size string `mapstructure:"size"` Image string `mapstructure:"image"` From a691a1521c5f722d7e3449fd84fcf40fa3183b6a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:16:40 -0700 Subject: [PATCH 35/85] website: update do docs --- .../docs/builders/digitalocean.html.markdown | 51 ++----------------- 1 file changed, 4 insertions(+), 47 deletions(-) diff --git a/website/source/docs/builders/digitalocean.html.markdown b/website/source/docs/builders/digitalocean.html.markdown index 5ffe1c668..28254b19c 100644 --- a/website/source/docs/builders/digitalocean.html.markdown +++ b/website/source/docs/builders/digitalocean.html.markdown @@ -24,31 +24,13 @@ There are many configuration options available for the builder. They are segmented below into two categories: required and optional parameters. Within each category, the available configuration keys are alphabetized. -### Required v1 api: +### Required: -* `api_key` (string) - The API key to use to access your account. You can - retrieve this on the "API" page visible after logging into your account - on DigitalOcean. - If not specified, Packer will use the environment variable - `DIGITALOCEAN_API_KEY`, if set. - -* `client_id` (string) - The client ID to use to access your account. You can - find this on the "API" page visible after logging into your account on - DigitalOcean. - If not specified, Packer will use the environment variable - `DIGITALOCEAN_CLIENT_ID`, if set. - -### Required v2 api: - -* `api_token` (string) - The client TOKEN to use to access your account. If it - specified, then use v2 api (current), if not then used old (v1) deprecated api. - Also it can be specified via environment variable `DIGITALOCEAN_API_TOKEN`, if set. +* `api_token` (string) - The client TOKEN to use to access your account. + It can also be specified via environment variable `DIGITALOCEAN_API_TOKEN`, if set. ### Optional: -* `api_url` (string) - API endpoint, by default use https://api.digitalocean.com - Also it can be specified via environment variable `DIGITALOCEAN_API_URL`, if set. - * `droplet_name` (string) - The name assigned to the droplet. DigitalOcean sets the hostname of the machine to this value. @@ -57,10 +39,6 @@ each category, the available configuration keys are alphabetized. defaults to 'ubuntu-12-04-x64' which is the slug for "Ubuntu 12.04.4 x64". See https://developers.digitalocean.com/documentation/v2/#list-all-images for details on how to get a list of the the accepted image names/slugs. -* `image_id` (integer) - The ID of the base image to use. This is the image that - will be used to launch a new droplet and provision it. - This setting is deprecated. Use `image` instead. - * `private_networking` (boolean) - Set to `true` to enable private networking for the droplet being created. This defaults to `false`, or not enabled. @@ -69,17 +47,10 @@ each category, the available configuration keys are alphabetized. This defaults to "nyc3", which is the slug for "New York 3". See https://developers.digitalocean.com/documentation/v2/#list-all-regions for the accepted region names/slugs. -* `region_id` (integer) - The ID of the region to launch the droplet in. Consequently, - this is the region where the snapshot will be available. - This setting is deprecated. Use `region` instead. - * `size` (string) - The name (or slug) of the droplet size to use. This defaults to "512mb", which is the slug for "512MB". See https://developers.digitalocean.com/documentation/v2/#list-all-sizes for the accepted size names/slugs. -* `size_id` (integer) - The ID of the droplet size to use. - This setting is deprecated. Use `size` instead. - * `snapshot_name` (string) - The name of the resulting snapshot that will appear in your account. This must be unique. To help make this unique, use a function like `timestamp` (see @@ -107,20 +78,6 @@ own access tokens: ```javascript { "type": "digitalocean", - "client_id": "YOUR CLIENT ID", - "api_key": "YOUR API KEY" + "api_token": "YOUR API KEY" } ``` - -## Finding Image, Region, and Size IDs - -Unfortunately, finding a list of available values for `image_id`, `region_id`, -and `size_id` is not easy at the moment. Basically, it has to be done through -the [DigitalOcean API](https://www.digitalocean.com/api_access) using the -`/images`, `/regions`, and `/sizes` endpoints. You can use `curl` for this -or request it in your browser. - -If you're comfortable installing RubyGems, [Tugboat](https://github.com/pearkes/tugboat) -is a fantastic DigitalOcean command-line client that has commands to -find the available images, regions, and sizes. For example, to see all the -global images, you can run `tugboat images --global`. From 9a393a560172e286cf9b9319a6689822b47c2783 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:18:38 -0700 Subject: [PATCH 36/85] builder/digitalocean: only list user images --- builder/digitalocean/step_snapshot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/digitalocean/step_snapshot.go b/builder/digitalocean/step_snapshot.go index 7ff384924..cfda7af20 100644 --- a/builder/digitalocean/step_snapshot.go +++ b/builder/digitalocean/step_snapshot.go @@ -37,7 +37,7 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { } log.Printf("Looking up snapshot ID for snapshot: %s", c.SnapshotName) - images, _, err := client.Images.List(nil) + images, _, err := client.Images.ListUser(&godo.ListOptions{PerPage: 200}) if err != nil { err := fmt.Errorf("Error looking up snapshot ID: %s", err) state.Put("error", err) From 2056fda4d3a91e670f628f554b0fcb0e3725bddd Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Wed, 10 Jun 2015 16:19:36 -0500 Subject: [PATCH 37/85] builder/amazon: Allow spaces in AMI names --- builder/amazon/common/template_funcs.go | 2 +- builder/amazon/common/template_funcs_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/template_funcs.go b/builder/amazon/common/template_funcs.go index 45f68af92..30d49fdb4 100644 --- a/builder/amazon/common/template_funcs.go +++ b/builder/amazon/common/template_funcs.go @@ -20,7 +20,7 @@ func isalphanumeric(b byte) bool { // Clean up AMI name by replacing invalid characters with "-" func templateCleanAMIName(s string) string { - allowed := []byte{'(', ')', ',', '/', '-', '_'} + allowed := []byte{'(', ')', ',', '/', '-', '_', ' '} b := []byte(s) newb := make([]byte, len(b)) for i, c := range b { diff --git a/builder/amazon/common/template_funcs_test.go b/builder/amazon/common/template_funcs_test.go index 0e8c568ef..e4126bf61 100644 --- a/builder/amazon/common/template_funcs_test.go +++ b/builder/amazon/common/template_funcs_test.go @@ -5,8 +5,8 @@ import ( ) func TestAMITemplatePrepare_clean(t *testing.T) { - origName := "AMZamz09(),/-_:&^$%" - expected := "AMZamz09(),/-_-----" + origName := "AMZamz09(),/-_:&^ $%" + expected := "AMZamz09(),/-_--- --" name := templateCleanAMIName(origName) From 311c9eb5c2b594312ba6e69ff47b71419657349c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 14:28:05 -0700 Subject: [PATCH 38/85] builder/digitalocean: fix unit tests --- builder/digitalocean/builder_test.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/builder/digitalocean/builder_test.go b/builder/digitalocean/builder_test.go index 8985aae2f..878431691 100644 --- a/builder/digitalocean/builder_test.go +++ b/builder/digitalocean/builder_test.go @@ -1,22 +1,15 @@ package digitalocean import ( - "github.com/mitchellh/packer/packer" - "os" "strconv" "testing" -) -func init() { - // Clear out the credential env vars - os.Setenv("DIGITALOCEAN_API_KEY", "") - os.Setenv("DIGITALOCEAN_CLIENT_ID", "") -} + "github.com/mitchellh/packer/packer" +) func testConfig() map[string]interface{} { return map[string]interface{}{ - "client_id": "foo", - "api_key": "bar", + "api_token": "bar", } } @@ -78,7 +71,6 @@ func TestBuilderPrepare_Region(t *testing.T) { expected := "sfo1" // Test set - config["region_id"] = 0 config["region"] = expected b = Builder{} warnings, err = b.Prepare(config) @@ -114,7 +106,6 @@ func TestBuilderPrepare_Size(t *testing.T) { expected := "1024mb" // Test set - config["size_id"] = 0 config["size"] = expected b = Builder{} warnings, err = b.Prepare(config) @@ -150,7 +141,6 @@ func TestBuilderPrepare_Image(t *testing.T) { expected := "ubuntu-14-04-x64" // Test set - config["image_id"] = 0 config["image"] = expected b = Builder{} warnings, err = b.Prepare(config) From be8443d50813919a3d9fc0e39135d0d570a6c968 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:49:27 -0700 Subject: [PATCH 39/85] update CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a33dbc92..cc2ea5217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## 0.8.0 (unreleased) +BACKWARDS INCOMPATIBILITIES: + + * The DigitalOcean builder no longer supports the v1 API which has been + deprecated for some time. Most configurations should continue to + work as long as you use the `api_token` field for auth. + FEATURES: * **New config function: `template_dir`**: The directory to the template From 5da56d2aa6622b5096cdb1c5efbeba0408c63aa0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:54:51 -0700 Subject: [PATCH 40/85] builder/digitalocean: image, region, etc. required --- builder/digitalocean/builder.go | 12 ----------- builder/digitalocean/builder_acc_test.go | 5 ++++- builder/digitalocean/builder_test.go | 24 ++++++--------------- builder/digitalocean/config.go | 27 +++++++++++++----------- 4 files changed, 25 insertions(+), 43 deletions(-) diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index 3ba7074a2..996eb2e09 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -14,18 +14,6 @@ import ( "golang.org/x/oauth2" ) -// see https://api.digitalocean.com/images/?client_id=[client_id]&api_key=[api_key] -// name="Ubuntu 12.04.4 x64", id=6374128, -const DefaultImage = "ubuntu-12-04-x64" - -// see https://api.digitalocean.com/regions/?client_id=[client_id]&api_key=[api_key] -// name="New York 3", id=8 -const DefaultRegion = "nyc3" - -// see https://api.digitalocean.com/sizes/?client_id=[client_id]&api_key=[api_key] -// name="512MB", id=66 (the smallest droplet size) -const DefaultSize = "512mb" - // The unique id for the builder const BuilderId = "pearkes.digitalocean" diff --git a/builder/digitalocean/builder_acc_test.go b/builder/digitalocean/builder_acc_test.go index 20e56b924..f9df863cb 100644 --- a/builder/digitalocean/builder_acc_test.go +++ b/builder/digitalocean/builder_acc_test.go @@ -24,7 +24,10 @@ func testAccPreCheck(t *testing.T) { const testBuilderAccBasic = ` { "builders": [{ - "type": "test" + "type": "test", + "region": "nyc2", + "size": "512mb", + "image": "ubuntu-12-04-x64" }] } ` diff --git a/builder/digitalocean/builder_test.go b/builder/digitalocean/builder_test.go index 878431691..3d2378f2d 100644 --- a/builder/digitalocean/builder_test.go +++ b/builder/digitalocean/builder_test.go @@ -60,12 +60,8 @@ func TestBuilderPrepare_Region(t *testing.T) { if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.Region != DefaultRegion { - t.Errorf("found %s, expected %s", b.config.Region, DefaultRegion) + if err == nil { + t.Fatalf("should error") } expected := "sfo1" @@ -95,12 +91,8 @@ func TestBuilderPrepare_Size(t *testing.T) { if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.Size != DefaultSize { - t.Errorf("found %s, expected %s", b.config.Size, DefaultSize) + if err == nil { + t.Fatalf("should error") } expected := "1024mb" @@ -130,12 +122,8 @@ func TestBuilderPrepare_Image(t *testing.T) { if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - if b.config.Image != DefaultImage { - t.Errorf("found %s, expected %s", b.config.Image, DefaultImage) + if err == nil { + t.Fatal("should error") } expected := "ubuntu-14-04-x64" diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 5defe89db..057138633 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -63,18 +63,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { c.APIToken = os.Getenv("DIGITALOCEAN_API_TOKEN") } - if c.Region == "" { - c.Region = DefaultRegion - } - - if c.Size == "" { - c.Size = DefaultSize - } - - if c.Image == "" { - c.Image = DefaultImage - } - if c.SnapshotName == "" { // Default to packer-{{ unix timestamp (utc) }} c.SnapshotName = "packer-{{timestamp}}" @@ -114,6 +102,21 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { errs, errors.New("api_token for auth must be specified")) } + if c.Region == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("region is required")) + } + + if c.Size == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("size is required")) + } + + if c.Image == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("image is required")) + } + sshTimeout, err := time.ParseDuration(c.RawSSHTimeout) if err != nil { errs = packer.MultiErrorAppend( From 910b16104ed5ed5d654c56f9e0749873fedfdd84 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:56:03 -0700 Subject: [PATCH 41/85] website: update required options for DO --- CHANGELOG.md | 1 + .../docs/builders/digitalocean.html.markdown | 30 +++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc2ea5217..fd57fb8c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ BACKWARDS INCOMPATIBILITIES: * The DigitalOcean builder no longer supports the v1 API which has been deprecated for some time. Most configurations should continue to work as long as you use the `api_token` field for auth. + * builder/digitalocean: `image`, `region`, and `size` are now required. FEATURES: diff --git a/website/source/docs/builders/digitalocean.html.markdown b/website/source/docs/builders/digitalocean.html.markdown index 28254b19c..34d11bedb 100644 --- a/website/source/docs/builders/digitalocean.html.markdown +++ b/website/source/docs/builders/digitalocean.html.markdown @@ -29,28 +29,25 @@ each category, the available configuration keys are alphabetized. * `api_token` (string) - The client TOKEN to use to access your account. It can also be specified via environment variable `DIGITALOCEAN_API_TOKEN`, if set. +* `image` (string) - The name (or slug) of the base image to use. This is the + image that will be used to launch a new droplet and provision it. + See https://developers.digitalocean.com/documentation/v2/#list-all-images for details on how to get a list of the the accepted image names/slugs. + +* `region` (string) - The name (or slug) of the region to launch the droplet in. + Consequently, this is the region where the snapshot will be available. + See https://developers.digitalocean.com/documentation/v2/#list-all-regions for the accepted region names/slugs. + +* `size` (string) - The name (or slug) of the droplet size to use. + See https://developers.digitalocean.com/documentation/v2/#list-all-sizes for the accepted size names/slugs. + ### Optional: * `droplet_name` (string) - The name assigned to the droplet. DigitalOcean sets the hostname of the machine to this value. -* `image` (string) - The name (or slug) of the base image to use. This is the - image that will be used to launch a new droplet and provision it. This - defaults to 'ubuntu-12-04-x64' which is the slug for "Ubuntu 12.04.4 x64". - See https://developers.digitalocean.com/documentation/v2/#list-all-images for details on how to get a list of the the accepted image names/slugs. - * `private_networking` (boolean) - Set to `true` to enable private networking for the droplet being created. This defaults to `false`, or not enabled. -* `region` (string) - The name (or slug) of the region to launch the droplet in. - Consequently, this is the region where the snapshot will be available. - This defaults to "nyc3", which is the slug for "New York 3". - See https://developers.digitalocean.com/documentation/v2/#list-all-regions for the accepted region names/slugs. - -* `size` (string) - The name (or slug) of the droplet size to use. - This defaults to "512mb", which is the slug for "512MB". - See https://developers.digitalocean.com/documentation/v2/#list-all-sizes for the accepted size names/slugs. - * `snapshot_name` (string) - The name of the resulting snapshot that will appear in your account. This must be unique. To help make this unique, use a function like `timestamp` (see @@ -78,6 +75,9 @@ own access tokens: ```javascript { "type": "digitalocean", - "api_token": "YOUR API KEY" + "api_token": "YOUR API KEY", + "image": "ubuntu-12-04-x64", + "region": "nyc2", + "size": "512mb" } ``` From 6b29c2d26a80289c720300cc0381613148c938ad Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:58:27 -0700 Subject: [PATCH 42/85] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd57fb8c3..357afa298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ FEATURES: IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] + * builder/digitalocean: Save SSH key to pwd if debug mode is on. [GH-1829] * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear From c9c9e2871c2cfc6b2e3b17c5c24eb58cc5a6f45c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 18:58:50 -0700 Subject: [PATCH 43/85] builder/digitalocean: fix build --- builder/digitalocean/builder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index c269da6e1..97569d0fe 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -4,6 +4,7 @@ package digitalocean import ( + "fmt" "log" "time" @@ -47,7 +48,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ &stepCreateSSHKey{ - Debug: b.config.PackerDebug, + Debug: b.config.PackerDebug, DebugKeyPath: fmt.Sprintf("do_%s.pem", b.config.PackerBuildName), }, new(stepCreateDroplet), From 5cfd26a0d336494f2273076b6f1cf2341bcec551 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 19:29:48 -0700 Subject: [PATCH 44/85] builder/digitalocean: user data support [GH-2113] --- builder/digitalocean/config.go | 1 + builder/digitalocean/step_create_droplet.go | 1 + website/source/docs/builders/digitalocean.html.markdown | 2 ++ 3 files changed, 4 insertions(+) diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 057138633..5621d64d7 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -26,6 +26,7 @@ type Config struct { PrivateNetworking bool `mapstructure:"private_networking"` SnapshotName string `mapstructure:"snapshot_name"` DropletName string `mapstructure:"droplet_name"` + UserData string `mapstructure:"user_data"` SSHUsername string `mapstructure:"ssh_username"` SSHPort uint `mapstructure:"ssh_port"` diff --git a/builder/digitalocean/step_create_droplet.go b/builder/digitalocean/step_create_droplet.go index 40ac8f0e9..aafd53622 100644 --- a/builder/digitalocean/step_create_droplet.go +++ b/builder/digitalocean/step_create_droplet.go @@ -31,6 +31,7 @@ func (s *stepCreateDroplet) Run(state multistep.StateBag) multistep.StepAction { godo.DropletCreateSSHKey{ID: int(sshKeyId)}, }, PrivateNetworking: c.PrivateNetworking, + UserData: c.UserData, }) if err != nil { err := fmt.Errorf("Error creating droplet: %s", err) diff --git a/website/source/docs/builders/digitalocean.html.markdown b/website/source/docs/builders/digitalocean.html.markdown index 34d11bedb..829424e3d 100644 --- a/website/source/docs/builders/digitalocean.html.markdown +++ b/website/source/docs/builders/digitalocean.html.markdown @@ -67,6 +67,8 @@ each category, the available configuration keys are alphabetized. for a droplet to enter a desired state (such as "active") before timing out. The default state timeout is "6m". +* `user_data` (string) - User data to launch with the Droplet. + ## Basic Example Here is a basic example. It is completely valid as soon as you enter your From 0e0cd28071942199858f2e9384368473c3c8d54a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 19:31:48 -0700 Subject: [PATCH 45/85] builder/digitalocean: fix failing unit tests --- builder/digitalocean/builder_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/builder/digitalocean/builder_test.go b/builder/digitalocean/builder_test.go index 3d2378f2d..22c9e3b50 100644 --- a/builder/digitalocean/builder_test.go +++ b/builder/digitalocean/builder_test.go @@ -10,6 +10,9 @@ import ( func testConfig() map[string]interface{} { return map[string]interface{}{ "api_token": "bar", + "region": "nyc2", + "size": "512mb", + "image": "foo", } } @@ -56,6 +59,7 @@ func TestBuilderPrepare_Region(t *testing.T) { config := testConfig() // Test default + delete(config, "region") warnings, err := b.Prepare(config) if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) @@ -87,6 +91,7 @@ func TestBuilderPrepare_Size(t *testing.T) { config := testConfig() // Test default + delete(config, "size") warnings, err := b.Prepare(config) if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) @@ -118,6 +123,7 @@ func TestBuilderPrepare_Image(t *testing.T) { config := testConfig() // Test default + delete(config, "image") warnings, err := b.Prepare(config) if len(warnings) > 0 { t.Fatalf("bad: %#v", warnings) From dcf140f99facb21ef1e177866c9cb41eb4349093 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 19:53:07 -0700 Subject: [PATCH 46/85] builder/digitalocean: more robust wait for pending --- builder/digitalocean/config.go | 7 +++- builder/digitalocean/step_power_off.go | 10 ++++++ builder/digitalocean/step_shutdown.go | 14 +++++++- builder/digitalocean/step_snapshot.go | 13 +++++++ builder/digitalocean/wait.go | 49 ++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 2 deletions(-) diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 057138633..dd1460583 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -64,8 +64,13 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { } if c.SnapshotName == "" { + def, err := interpolate.Render("packer-{{timestamp}}", nil) + if err != nil { + panic(err) + } + // Default to packer-{{ unix timestamp (utc) }} - c.SnapshotName = "packer-{{timestamp}}" + c.SnapshotName = def } if c.DropletName == "" { diff --git a/builder/digitalocean/step_power_off.go b/builder/digitalocean/step_power_off.go index 3d547e8c2..94891e227 100644 --- a/builder/digitalocean/step_power_off.go +++ b/builder/digitalocean/step_power_off.go @@ -3,6 +3,7 @@ package digitalocean import ( "fmt" "log" + "time" "github.com/digitalocean/godo" "github.com/mitchellh/multistep" @@ -48,6 +49,15 @@ func (s *stepPowerOff) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } + // Wait for the droplet to become unlocked for future steps + if err := waitForDropletUnlocked(client, dropletId, 2*time.Minute); err != nil { + // If we get an error the first time, actually report it + err := fmt.Errorf("Error powering off droplet: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + return multistep.ActionContinue } diff --git a/builder/digitalocean/step_shutdown.go b/builder/digitalocean/step_shutdown.go index 602f3e690..da04aee33 100644 --- a/builder/digitalocean/step_shutdown.go +++ b/builder/digitalocean/step_shutdown.go @@ -65,7 +65,19 @@ func (s *stepShutdown) Run(state multistep.StateBag) multistep.StepAction { err = waitForDropletState("off", dropletId, client, 2*time.Minute) if err != nil { - log.Printf("Error waiting for graceful off: %s", err) + // If we get an error the first time, actually report it + err := fmt.Errorf("Error shutting down droplet: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if err := waitForDropletUnlocked(client, dropletId, 2*time.Minute); err != nil { + // If we get an error the first time, actually report it + err := fmt.Errorf("Error shutting down droplet: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt } return multistep.ActionContinue diff --git a/builder/digitalocean/step_snapshot.go b/builder/digitalocean/step_snapshot.go index cfda7af20..f6902b8c5 100644 --- a/builder/digitalocean/step_snapshot.go +++ b/builder/digitalocean/step_snapshot.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "log" + "time" "github.com/digitalocean/godo" "github.com/mitchellh/multistep" @@ -27,6 +28,18 @@ func (s *stepSnapshot) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } + // Wait for the droplet to become unlocked first. For snapshots + // this can end up taking quite a long time, so we hardcode this to + // 10 minutes. + if err := waitForDropletUnlocked(client, dropletId, 10*time.Minute); err != nil { + // If we get an error the first time, actually report it + err := fmt.Errorf("Error shutting down droplet: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // With the pending state over, verify that we're in the active state ui.Say("Waiting for snapshot to complete...") err = waitForDropletState("active", dropletId, client, c.stateTimeout) if err != nil { diff --git a/builder/digitalocean/wait.go b/builder/digitalocean/wait.go index 3d299d433..a41bbb3ed 100644 --- a/builder/digitalocean/wait.go +++ b/builder/digitalocean/wait.go @@ -8,6 +8,55 @@ import ( "github.com/digitalocean/godo" ) +// waitForDropletUnlocked waits for the Droplet to be unlocked to +// avoid "pending" errors when making state changes. +func waitForDropletUnlocked( + client *godo.Client, dropletId int, timeout time.Duration) error { + done := make(chan struct{}) + defer close(done) + + result := make(chan error, 1) + go func() { + attempts := 0 + for { + attempts += 1 + + log.Printf("[DEBUG] Checking droplet lock state... (attempt: %d)", attempts) + droplet, _, err := client.Droplets.Get(dropletId) + if err != nil { + result <- err + return + } + + if !droplet.Locked { + result <- nil + return + } + + // Wait 3 seconds in between + time.Sleep(3 * time.Second) + + // Verify we shouldn't exit + select { + case <-done: + // We finished, so just exit the goroutine + return + default: + // Keep going + } + } + }() + + log.Printf("[DEBUG] Waiting for up to %d seconds for droplet to unlock", timeout/time.Second) + select { + case err := <-result: + return err + case <-time.After(timeout): + return fmt.Errorf( + "Timeout while waiting to for droplet to unlock") + } +} + // waitForState simply blocks until the droplet is in // a state we expect, while eventually timing out. func waitForDropletState( From c9d308c5d7f4a26c620fc5be10df045c018b7505 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 19:53:52 -0700 Subject: [PATCH 47/85] update CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 357afa298..b79d6a4bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,8 @@ BUG FIXES: * builder/amazon/chroot: Retry waiting for disk attachments [GH-2046] * builder/amazon/instance: Use `-i` in sudo commands so PATH is inherited. [GH-1930] * builder/amazon/instance: Use `--region` flag for bundle upload command. [GH-1931] + * builder/digitalocean: Wait for droplet to unlock before changing state, + should lower the "pending event" errors. * builder/digitalocean: Ignore invalid fields from the ever-changing v2 API * builder/digitalocean: Private images can be used as a source [GH-1792] * builder/docker: Fixed hang on prompt while copying script From 03032c26fbef7d2a4ba79fc06d437b10f0a4eb18 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 20:23:39 -0700 Subject: [PATCH 48/85] ignore non-exe plugins on Windows [GH-2173] --- config.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config.go b/config.go index 745922b1e..20e8929c3 100644 --- a/config.go +++ b/config.go @@ -6,6 +6,7 @@ import ( "log" "os/exec" "path/filepath" + "runtime" "strings" "github.com/mitchellh/osext" @@ -172,6 +173,15 @@ func (c *config) discoverSingle(glob string, m *map[string]string) error { for _, match := range matches { file := filepath.Base(match) + // One Windows, ignore any plugins that don't end in .exe. + // We could do a full PATHEXT parse, but this is probably good enough. + if runtime.GOOS == "windows" && filepath.Ext(file) != "exe" { + log.Printf( + "[DEBUG] Ignoring plugin match %s, no exe extension", + match) + continue + } + // If the filename has a ".", trim up to there if idx := strings.Index(file, "."); idx >= 0 { file = file[:idx] From 9514be0df0613f4c7c6dd8e0b25f62f026ace9d3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 20:24:34 -0700 Subject: [PATCH 49/85] lowercase the extension of plugins just in case --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 20e8929c3..2ebc66422 100644 --- a/config.go +++ b/config.go @@ -175,7 +175,7 @@ func (c *config) discoverSingle(glob string, m *map[string]string) error { // One Windows, ignore any plugins that don't end in .exe. // We could do a full PATHEXT parse, but this is probably good enough. - if runtime.GOOS == "windows" && filepath.Ext(file) != "exe" { + if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != "exe" { log.Printf( "[DEBUG] Ignoring plugin match %s, no exe extension", match) From 9c1e6bc478c150920a2a6433fddbf6808b0f1a12 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 20:30:00 -0700 Subject: [PATCH 50/85] website: clarify shutdown command [GH-2011] --- website/source/docs/builders/virtualbox-iso.html.markdown | 8 +++++--- website/source/docs/builders/virtualbox-ovf.html.markdown | 8 +++++--- website/source/docs/builders/vmware-vmx.html.markdown | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/website/source/docs/builders/virtualbox-iso.html.markdown b/website/source/docs/builders/virtualbox-iso.html.markdown index 03a3fb23a..07204a33c 100644 --- a/website/source/docs/builders/virtualbox-iso.html.markdown +++ b/website/source/docs/builders/virtualbox-iso.html.markdown @@ -179,9 +179,11 @@ each category, the available options are alphabetized and described. By default this is "output-BUILDNAME" where "BUILDNAME" is the name of the build. -* `shutdown_command` (string) - The command to use to gracefully shut down - the machine once all the provisioning is done. By default this is an empty - string, which tells Packer to just forcefully shut down the machine. +* `shutdown_command` (string) - The command to use to gracefully shut down the machine once all + the provisioning is done. By default this is an empty string, which tells Packer to just + forcefully shut down the machine unless a shutdown command takes place inside script so this may + safely be omitted. If one or more scripts require a reboot it is suggested to leave this blank + since reboots may fail and specify the final shutdown command in your last script. * `shutdown_timeout` (string) - The amount of time to wait after executing the `shutdown_command` for the virtual machine to actually shut down. diff --git a/website/source/docs/builders/virtualbox-ovf.html.markdown b/website/source/docs/builders/virtualbox-ovf.html.markdown index 88a92b674..9635d0e60 100644 --- a/website/source/docs/builders/virtualbox-ovf.html.markdown +++ b/website/source/docs/builders/virtualbox-ovf.html.markdown @@ -154,9 +154,11 @@ each category, the available options are alphabetized and described. By default this is "output-BUILDNAME" where "BUILDNAME" is the name of the build. -* `shutdown_command` (string) - The command to use to gracefully shut down - the machine once all the provisioning is done. By default this is an empty - string, which tells Packer to just forcefully shut down the machine. +* `shutdown_command` (string) - The command to use to gracefully shut down the machine once all + the provisioning is done. By default this is an empty string, which tells Packer to just + forcefully shut down the machine unless a shutdown command takes place inside script so this may + safely be omitted. If one or more scripts require a reboot it is suggested to leave this blank + since reboots may fail and specify the final shutdown command in your last script. * `shutdown_timeout` (string) - The amount of time to wait after executing the `shutdown_command` for the virtual machine to actually shut down. diff --git a/website/source/docs/builders/vmware-vmx.html.markdown b/website/source/docs/builders/vmware-vmx.html.markdown index 399bab8a9..bbdd8925c 100644 --- a/website/source/docs/builders/vmware-vmx.html.markdown +++ b/website/source/docs/builders/vmware-vmx.html.markdown @@ -109,9 +109,11 @@ each category, the available options are alphabetized and described. By default this is "output-BUILDNAME" where "BUILDNAME" is the name of the build. -* `shutdown_command` (string) - The command to use to gracefully shut down - the machine once all the provisioning is done. By default this is an empty - string, which tells Packer to just forcefully shut down the machine. +* `shutdown_command` (string) - The command to use to gracefully shut down the machine once all + the provisioning is done. By default this is an empty string, which tells Packer to just + forcefully shut down the machine unless a shutdown command takes place inside script so this may + safely be omitted. If one or more scripts require a reboot it is suggested to leave this blank + since reboots may fail and specify the final shutdown command in your last script. * `shutdown_timeout` (string) - The amount of time to wait after executing the `shutdown_command` for the virtual machine to actually shut down. From 9c1e461402bb88c32e08c8019e7f9816e36eb2a0 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 20:33:00 -0700 Subject: [PATCH 51/85] website: document chef_environment for chef-solo --- website/source/docs/provisioners/chef-solo.html.markdown | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/source/docs/provisioners/chef-solo.html.markdown b/website/source/docs/provisioners/chef-solo.html.markdown index 178ba0c62..3a76c5514 100644 --- a/website/source/docs/provisioners/chef-solo.html.markdown +++ b/website/source/docs/provisioners/chef-solo.html.markdown @@ -34,6 +34,9 @@ The example below is fully functional and expects cookbooks in the The reference of available configuration options is listed below. No configuration is actually required, but at least `run_list` is recommended. +* `chef_environment` (string) - The name of the `chef_environment` sent to the + Chef server. By default this is empty and will not use an environment + * `config_template` (string) - Path to a template that will be used for the Chef configuration file. By default Packer only sets configuration it needs to match the settings set in the provisioner configuration. If From 952077ccb0b1cfe189a43b8001d6dac3ea29b64c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 21:02:22 -0700 Subject: [PATCH 52/85] provisioner/shell: set -e for inline [GH-2069] --- provisioner/shell/provisioner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index 48904710d..8b6ceb705 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -184,6 +184,7 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { // Write our contents to it writer := bufio.NewWriter(tf) writer.WriteString(fmt.Sprintf("#!%s\n", p.config.InlineShebang)) + writer.WriteString("set -e\n") for _, command := range p.config.Inline { if _, err := writer.WriteString(command + "\n"); err != nil { return fmt.Errorf("Error preparing shell script: %s", err) From a995df352e7a6967c7e40d1a165b89af98a4abd3 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Wed, 10 Jun 2015 21:18:46 -0700 Subject: [PATCH 53/85] provisioner/shell: uploaded script should be 0755 [GH-1708] --- provisioner/shell/provisioner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index 48904710d..f07c91936 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -247,11 +247,11 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { } cmd = &packer.RemoteCmd{ - Command: fmt.Sprintf("chmod 0777 %s", p.config.RemotePath), + Command: fmt.Sprintf("chmod 0755 %s", p.config.RemotePath), } if err := comm.Start(cmd); err != nil { return fmt.Errorf( - "Error chmodding script file to 0777 in remote "+ + "Error chmodding script file to 0755 in remote "+ "machine: %s", err) } cmd.Wait() From e8f846e47e93669a9eee2d1daaa4d649a71667ab Mon Sep 17 00:00:00 2001 From: Clint Date: Thu, 11 Jun 2015 08:50:48 -0500 Subject: [PATCH 54/85] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b79d6a4bb..abb0d46d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ IMPROVEMENTS: BUG FIXES: * core: Fix potential panic for post-processor plugin exits [GH-2098] + * builder/amazon: Allow spaces in AMI names when using `clean_ami_name` [GH-2182] * builder/amazon: Remove deprecated ec2-upload-bundle paramger [GH-1931] * builder/amazon: Retry finding created instance for eventual consistency. [GH-2129] From 16f8866728d81acf43925427d2b89db581a55fb1 Mon Sep 17 00:00:00 2001 From: Clint Date: Thu, 11 Jun 2015 10:57:07 -0500 Subject: [PATCH 55/85] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index abb0d46d0..4705c3523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ BUG FIXES: * core: Fix potential panic for post-processor plugin exits [GH-2098] * builder/amazon: Allow spaces in AMI names when using `clean_ami_name` [GH-2182] * builder/amazon: Remove deprecated ec2-upload-bundle paramger [GH-1931] + * builder/amazon: Use IAM Profile to upload bundle if provided [GH-1985] * builder/amazon: Retry finding created instance for eventual consistency. [GH-2129] * builder/amazon: If no AZ is specified, use AZ chosen automatically by From 724d591ba4f29817b4f97085b463ba728936bfe1 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Thu, 11 Jun 2015 14:02:00 -0500 Subject: [PATCH 56/85] documentation: subnet_id is required for non-default VPC --- website/Gemfile.lock | 9 ++++++--- website/source/docs/builders/amazon-ebs.html.markdown | 3 ++- .../source/docs/builders/amazon-instance.html.markdown | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/website/Gemfile.lock b/website/Gemfile.lock index 7366999a6..b49383f77 100644 --- a/website/Gemfile.lock +++ b/website/Gemfile.lock @@ -79,18 +79,18 @@ GEM celluloid (~> 0.16.0) rb-fsevent (>= 0.9.3) rb-inotify (>= 0.9) - middleman (3.3.13) + middleman (3.3.12) coffee-script (~> 2.2) compass (>= 1.0.0, < 2.0.0) compass-import-once (= 1.0.5) execjs (~> 2.0) haml (>= 4.0.5) kramdown (~> 1.2) - middleman-core (= 3.3.13) + middleman-core (= 3.3.12) middleman-sprockets (>= 3.1.2) sass (>= 3.4.0, < 4.0) uglifier (~> 2.5) - middleman-core (3.3.13) + middleman-core (3.3.12) activesupport (~> 4.1.0) bundler (~> 1.1) erubis @@ -175,3 +175,6 @@ PLATFORMS DEPENDENCIES middleman-hashicorp! + +BUNDLED WITH + 1.10.2 diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index 5e2c31e90..0ff9522df 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -144,7 +144,8 @@ each category, the available configuration keys are alphabetized. or "5m". The default SSH timeout is "5m", or five minutes. * `subnet_id` (string) - If using VPC, the ID of the subnet, such as - "subnet-12345def", where Packer will launch the EC2 instance. + "subnet-12345def", where Packer will launch the EC2 instance. This field is + required if you are using an non-default VPC. * `tags` (object of key/value strings) - Tags applied to the AMI. diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown index 8360322de..ae5fbff27 100644 --- a/website/source/docs/builders/amazon-instance.html.markdown +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -184,7 +184,8 @@ each category, the available configuration keys are alphabetized. or "5m". The default SSH timeout is "5m", or five minutes. * `subnet_id` (string) - If using VPC, the ID of the subnet, such as - "subnet-12345def", where Packer will launch the EC2 instance. + "subnet-12345def", where Packer will launch the EC2 instance. This field is + required if you are using an non-default VPC. * `tags` (object of key/value strings) - Tags applied to the AMI. From e88fa43cfe9bdb785176c6b9c1944ebecb43157b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 16:45:24 -0400 Subject: [PATCH 57/85] find proper extension --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 2ebc66422..efb4e7d31 100644 --- a/config.go +++ b/config.go @@ -175,7 +175,7 @@ func (c *config) discoverSingle(glob string, m *map[string]string) error { // One Windows, ignore any plugins that don't end in .exe. // We could do a full PATHEXT parse, but this is probably good enough. - if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != "exe" { + if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" { log.Printf( "[DEBUG] Ignoring plugin match %s, no exe extension", match) From ae386d0a36c1485276d76ac8a7c30b8fb77c97dd Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 16:55:32 -0400 Subject: [PATCH 58/85] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4705c3523..bef4170f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ BUG FIXES: * post-processor/vagrant-cloud: Don't delete version on error [GH-2014] * provisioner/puppet-masterless: Allow manifest_file to be a directory * provisioner/salt-masterless: Add `--retcode-passthrough` to salt-call + * provisioner/shell: chmod executable script to 0755, not 0777 [GH-1708] ## 0.7.5 (December 9, 2014) From 8ca8bd7866e29bfcf38e84c5393146922e5a404e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 16:57:44 -0400 Subject: [PATCH 59/85] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bef4170f1..ded839503 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ IMPROVEMENTS: * core: Interrupt handling for SIGTERM signal as well. [GH-1858] * builder/digitalocean: Save SSH key to pwd if debug mode is on. [GH-1829] + * builder/digitalocean: User data support [GH-2113] * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear From 3ed73852be35e04cb69c05459a298e7b1eb4a37c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 17:19:23 -0400 Subject: [PATCH 60/85] provisioner/shell: set -e on the shebang itself --- provisioner/shell/provisioner.go | 3 +-- website/source/docs/provisioners/shell.html.markdown | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/provisioner/shell/provisioner.go b/provisioner/shell/provisioner.go index 8b6ceb705..0d55c981b 100644 --- a/provisioner/shell/provisioner.go +++ b/provisioner/shell/provisioner.go @@ -94,7 +94,7 @@ func (p *Provisioner) Prepare(raws ...interface{}) error { } if p.config.InlineShebang == "" { - p.config.InlineShebang = "/bin/sh" + p.config.InlineShebang = "/bin/sh -e" } if p.config.RawStartRetryTimeout == "" { @@ -184,7 +184,6 @@ func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error { // Write our contents to it writer := bufio.NewWriter(tf) writer.WriteString(fmt.Sprintf("#!%s\n", p.config.InlineShebang)) - writer.WriteString("set -e\n") for _, command := range p.config.Inline { if _, err := writer.WriteString(command + "\n"); err != nil { return fmt.Errorf("Error preparing shell script: %s", err) diff --git a/website/source/docs/provisioners/shell.html.markdown b/website/source/docs/provisioners/shell.html.markdown index 7fcbe885b..e57910cb0 100644 --- a/website/source/docs/provisioners/shell.html.markdown +++ b/website/source/docs/provisioners/shell.html.markdown @@ -66,8 +66,10 @@ Optional parameters: * `inline_shebang` (string) - The [shebang](http://en.wikipedia.org/wiki/Shebang_%28Unix%29) value to use when - running commands specified by `inline`. By default, this is `/bin/sh`. + running commands specified by `inline`. By default, this is `/bin/sh -e`. If you're not using `inline`, then this configuration has no effect. + **Important:** If you customize this, be sure to include something like + the `-e` flag, otherwise individual steps failing won't fail the provisioner. * `remote_path` (string) - The path where the script will be uploaded to in the machine. This defaults to "/tmp/script.sh". This value must be From 2a912f7013703e263816513914ec374a9edaba6a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 17:20:13 -0400 Subject: [PATCH 61/85] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ded839503..207920b3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ BUG FIXES: * provisioner/puppet-masterless: Allow manifest_file to be a directory * provisioner/salt-masterless: Add `--retcode-passthrough` to salt-call * provisioner/shell: chmod executable script to 0755, not 0777 [GH-1708] + * provisioner/shell: inline commands failing will fail the provisioner [GH-2069] ## 0.7.5 (December 9, 2014) From 04e174fae899e80cb1d5d5a9b558e6c23a01a930 Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Thu, 11 Jun 2015 16:21:29 -0500 Subject: [PATCH 62/85] builder/amazon: Properly return error code on ssh errors --- common/step_connect_ssh.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/common/step_connect_ssh.go b/common/step_connect_ssh.go index c3d8aac2d..d45767ee5 100644 --- a/common/step_connect_ssh.go +++ b/common/step_connect_ssh.go @@ -3,13 +3,14 @@ package common import ( "errors" "fmt" + "log" + "strings" + "time" + "github.com/mitchellh/multistep" "github.com/mitchellh/packer/communicator/ssh" "github.com/mitchellh/packer/packer" gossh "golang.org/x/crypto/ssh" - "log" - "strings" - "time" ) // StepConnectSSH is a multistep Step implementation that waits for SSH @@ -64,6 +65,7 @@ WaitLoop: case <-waitDone: if err != nil { ui.Error(fmt.Sprintf("Error waiting for SSH: %s", err)) + state.Put("error", err) return multistep.ActionHalt } From 7830d78d063f5ade6d6b9b929f8b2c2a1a06999e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 17:24:02 -0400 Subject: [PATCH 63/85] provisioner/shell: fix tests --- provisioner/shell/provisioner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provisioner/shell/provisioner_test.go b/provisioner/shell/provisioner_test.go index 4fd5a5cad..54c41c956 100644 --- a/provisioner/shell/provisioner_test.go +++ b/provisioner/shell/provisioner_test.go @@ -45,7 +45,7 @@ func TestProvisionerPrepare_InlineShebang(t *testing.T) { t.Fatalf("should not have error: %s", err) } - if p.config.InlineShebang != "/bin/sh" { + if p.config.InlineShebang != "/bin/sh -e" { t.Fatalf("bad value: %s", p.config.InlineShebang) } From 25f5d6dba318d4894ca83e9fdc8a2fcd10273826 Mon Sep 17 00:00:00 2001 From: Clint Date: Thu, 11 Jun 2015 16:29:15 -0500 Subject: [PATCH 64/85] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 207920b3a..b1bc247f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ BUG FIXES: * builder/amazon: Allow spaces in AMI names when using `clean_ami_name` [GH-2182] * builder/amazon: Remove deprecated ec2-upload-bundle paramger [GH-1931] * builder/amazon: Use IAM Profile to upload bundle if provided [GH-1985] + * builder/amazon: Use correct exit code after SSH authentication failed [GH-2004] * builder/amazon: Retry finding created instance for eventual consistency. [GH-2129] * builder/amazon: If no AZ is specified, use AZ chosen automatically by From 17555ff21ad8c00f03101dd25743b25e716606a7 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 17:38:25 -0400 Subject: [PATCH 65/85] update version for dev --- version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version.go b/version.go index f7e44a1d0..dbfc8da12 100644 --- a/version.go +++ b/version.go @@ -4,9 +4,9 @@ package main var GitCommit string // The main version number that is being run at the moment. -const Version = "0.7.5" +const Version = "0.8.0" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -const VersionPrerelease = "" +const VersionPrerelease = "dev" From c903579aaacc66c01c893e4dad21480f1188f415 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 11 Jun 2015 23:43:36 -0400 Subject: [PATCH 66/85] builder/openstack-new --- builder/openstack-new/access_config.go | 109 ++++++++++++++ builder/openstack-new/artifact.go | 47 ++++++ builder/openstack-new/artifact_test.go | 35 +++++ builder/openstack-new/builder.go | 134 ++++++++++++++++++ builder/openstack-new/builder_test.go | 94 ++++++++++++ builder/openstack-new/image_config.go | 25 ++++ builder/openstack-new/image_config_test.go | 23 +++ builder/openstack-new/run_config.go | 75 ++++++++++ builder/openstack-new/run_config_test.go | 88 ++++++++++++ builder/openstack-new/server.go | 100 +++++++++++++ builder/openstack-new/ssh.go | 89 ++++++++++++ builder/openstack-new/step_allocate_ip.go | 94 ++++++++++++ builder/openstack-new/step_create_image.go | 82 +++++++++++ builder/openstack-new/step_key_pair.go | 106 ++++++++++++++ .../openstack-new/step_run_source_server.go | 110 ++++++++++++++ .../step_wait_for_rackconnect.go | 52 +++++++ plugin/builder-openstack-new/main.go | 15 ++ 17 files changed, 1278 insertions(+) create mode 100644 builder/openstack-new/access_config.go create mode 100644 builder/openstack-new/artifact.go create mode 100644 builder/openstack-new/artifact_test.go create mode 100644 builder/openstack-new/builder.go create mode 100644 builder/openstack-new/builder_test.go create mode 100644 builder/openstack-new/image_config.go create mode 100644 builder/openstack-new/image_config_test.go create mode 100644 builder/openstack-new/run_config.go create mode 100644 builder/openstack-new/run_config_test.go create mode 100644 builder/openstack-new/server.go create mode 100644 builder/openstack-new/ssh.go create mode 100644 builder/openstack-new/step_allocate_ip.go create mode 100644 builder/openstack-new/step_create_image.go create mode 100644 builder/openstack-new/step_key_pair.go create mode 100644 builder/openstack-new/step_run_source_server.go create mode 100644 builder/openstack-new/step_wait_for_rackconnect.go create mode 100644 plugin/builder-openstack-new/main.go diff --git a/builder/openstack-new/access_config.go b/builder/openstack-new/access_config.go new file mode 100644 index 000000000..e0f962c50 --- /dev/null +++ b/builder/openstack-new/access_config.go @@ -0,0 +1,109 @@ +package openstack + +import ( + "crypto/tls" + "fmt" + "net/http" + "os" + + "github.com/mitchellh/packer/template/interpolate" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack" +) + +// AccessConfig is for common configuration related to openstack access +type AccessConfig struct { + Username string `mapstructure:"username"` + UserID string `mapstructure:"user_id"` + Password string `mapstructure:"password"` + APIKey string `mapstructure:"api_key"` + IdentityEndpoint string `mapstructure:"identity_endpoint"` + TenantID string `mapstructure:"tenant_id"` + TenantName string `mapstructure:"tenant_name"` + DomainID string `mapstructure:"domain_id"` + DomainName string `mapstructure:"domain_name"` + Insecure bool `mapstructure:"insecure"` + Region string `mapstructure:"region"` + EndpointType string `mapstructure:"endpoint_type"` + + osClient *gophercloud.ProviderClient +} + +func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { + if c.EndpointType != "internal" && c.EndpointType != "internalURL" && + c.EndpointType != "admin" && c.EndpointType != "adminURL" && + c.EndpointType != "public" && c.EndpointType != "publicURL" && + c.EndpointType != "" { + return []error{fmt.Errorf("Invalid endpoint type provided")} + } + + if c.Region == "" { + c.Region = os.Getenv("OS_REGION_NAME") + } + + // Get as much as possible from the end + ao, err := openstack.AuthOptionsFromEnv() + if err != nil { + return []error{err} + } + + // Override values if we have them in our config + overrides := []struct { + From, To *string + }{ + {&c.Username, &ao.Username}, + {&c.UserID, &ao.UserID}, + {&c.Password, &ao.Password}, + {&c.APIKey, &ao.APIKey}, + {&c.IdentityEndpoint, &ao.IdentityEndpoint}, + {&c.TenantID, &ao.TenantID}, + {&c.TenantName, &ao.TenantName}, + {&c.DomainID, &ao.DomainID}, + {&c.DomainName, &ao.DomainName}, + } + for _, s := range overrides { + if *s.From != "" { + *s.To = *s.From + } + } + + // Build the client itself + client, err := openstack.NewClient(ao.IdentityEndpoint) + if err != nil { + return []error{err} + } + + // If we have insecure set, then create a custom HTTP client that + // ignores SSL errors. + if c.Insecure { + config := &tls.Config{InsecureSkipVerify: true} + transport := &http.Transport{TLSClientConfig: config} + client.HTTPClient.Transport = transport + } + + // Auth + err = openstack.Authenticate(client, ao) + if err != nil { + return []error{err} + } + + c.osClient = client + return nil +} + +func (c *AccessConfig) computeV2Client() (*gophercloud.ServiceClient, error) { + return openstack.NewComputeV2(c.osClient, gophercloud.EndpointOpts{ + Region: c.Region, + Availability: c.getEndpointType(), + }) +} + +func (c *AccessConfig) getEndpointType() gophercloud.Availability { + if c.EndpointType == "internal" || c.EndpointType == "internalURL" { + return gophercloud.AvailabilityInternal + } + if c.EndpointType == "admin" || c.EndpointType == "adminURL" { + return gophercloud.AvailabilityAdmin + } + return gophercloud.AvailabilityPublic +} diff --git a/builder/openstack-new/artifact.go b/builder/openstack-new/artifact.go new file mode 100644 index 000000000..aa60d2641 --- /dev/null +++ b/builder/openstack-new/artifact.go @@ -0,0 +1,47 @@ +package openstack + +import ( + "fmt" + "log" + + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/images" +) + +// Artifact is an artifact implementation that contains built images. +type Artifact struct { + // ImageId of built image + ImageId string + + // BuilderId is the unique ID for the builder that created this image + BuilderIdValue string + + // OpenStack connection for performing API stuff. + Client *gophercloud.ServiceClient +} + +func (a *Artifact) BuilderId() string { + return a.BuilderIdValue +} + +func (*Artifact) Files() []string { + // We have no files + return nil +} + +func (a *Artifact) Id() string { + return a.ImageId +} + +func (a *Artifact) String() string { + return fmt.Sprintf("An image was created: %v", a.ImageId) +} + +func (a *Artifact) State(name string) interface{} { + return nil +} + +func (a *Artifact) Destroy() error { + log.Printf("Destroying image: %s", a.ImageId) + return images.Delete(a.Client, a.ImageId).ExtractErr() +} diff --git a/builder/openstack-new/artifact_test.go b/builder/openstack-new/artifact_test.go new file mode 100644 index 000000000..313fea7cf --- /dev/null +++ b/builder/openstack-new/artifact_test.go @@ -0,0 +1,35 @@ +package openstack + +import ( + "github.com/mitchellh/packer/packer" + "testing" +) + +func TestArtifact_Impl(t *testing.T) { + var _ packer.Artifact = new(Artifact) +} + +func TestArtifactId(t *testing.T) { + expected := `b8cdf55b-c916-40bd-b190-389ec144c4ed` + + a := &Artifact{ + ImageId: "b8cdf55b-c916-40bd-b190-389ec144c4ed", + } + + result := a.Id() + if result != expected { + t.Fatalf("bad: %s", result) + } +} + +func TestArtifactString(t *testing.T) { + expected := "An image was created: b8cdf55b-c916-40bd-b190-389ec144c4ed" + + a := &Artifact{ + ImageId: "b8cdf55b-c916-40bd-b190-389ec144c4ed", + } + result := a.String() + if result != expected { + t.Fatalf("bad: %s", result) + } +} diff --git a/builder/openstack-new/builder.go b/builder/openstack-new/builder.go new file mode 100644 index 000000000..bebb28452 --- /dev/null +++ b/builder/openstack-new/builder.go @@ -0,0 +1,134 @@ +// The openstack package contains a packer.Builder implementation that +// builds Images for openstack. + +package openstack + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/common" + "log" + + "github.com/mitchellh/packer/helper/config" + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/template/interpolate" +) + +// The unique ID for this builder +const BuilderId = "mitchellh.openstack" + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + AccessConfig `mapstructure:",squash"` + ImageConfig `mapstructure:",squash"` + RunConfig `mapstructure:",squash"` + + ctx interpolate.Context +} + +type Builder struct { + config Config + runner multistep.Runner +} + +func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { + err := config.Decode(&b.config, &config.DecodeOpts{ + Interpolate: true, + }, raws...) + if err != nil { + return nil, err + } + + // Accumulate any errors + var errs *packer.MultiError + errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) + errs = packer.MultiErrorAppend(errs, b.config.ImageConfig.Prepare(&b.config.ctx)...) + errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...) + + if errs != nil && len(errs.Errors) > 0 { + return nil, errs + } + + log.Println(common.ScrubConfig(b.config, b.config.Password)) + return nil, nil +} + +func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { + computeClient, err := b.config.computeV2Client() + if err != nil { + return nil, fmt.Errorf("Error initializing compute client: %s", err) + } + + // Setup the state bag and initial state for the steps + state := new(multistep.BasicStateBag) + state.Put("config", b.config) + state.Put("hook", hook) + state.Put("ui", ui) + + // Build the steps + steps := []multistep.Step{ + &StepKeyPair{ + Debug: b.config.PackerDebug, + DebugKeyPath: fmt.Sprintf("os_%s.pem", b.config.PackerBuildName), + }, + &StepRunSourceServer{ + Name: b.config.ImageName, + Flavor: b.config.Flavor, + SourceImage: b.config.SourceImage, + SecurityGroups: b.config.SecurityGroups, + Networks: b.config.Networks, + }, + &StepWaitForRackConnect{ + Wait: b.config.RackconnectWait, + }, + &StepAllocateIp{ + FloatingIpPool: b.config.FloatingIpPool, + FloatingIp: b.config.FloatingIp, + }, + &common.StepConnectSSH{ + SSHAddress: SSHAddress(computeClient, b.config.SSHInterface, b.config.SSHPort), + SSHConfig: SSHConfig(b.config.SSHUsername), + SSHWaitTimeout: b.config.SSHTimeout(), + }, + &common.StepProvision{}, + &stepCreateImage{}, + } + + // Run! + if b.config.PackerDebug { + b.runner = &multistep.DebugRunner{ + Steps: steps, + PauseFn: common.MultistepDebugFn(ui), + } + } else { + b.runner = &multistep.BasicRunner{Steps: steps} + } + + b.runner.Run(state) + + // If there was an error, return that + if rawErr, ok := state.GetOk("error"); ok { + return nil, rawErr.(error) + } + + // If there are no images, then just return + if _, ok := state.GetOk("image"); !ok { + return nil, nil + } + + // Build the artifact and return it + artifact := &Artifact{ + ImageId: state.Get("image").(string), + BuilderIdValue: BuilderId, + Client: computeClient, + } + + return artifact, nil +} + +func (b *Builder) Cancel() { + if b.runner != nil { + log.Println("Cancelling the step runner...") + b.runner.Cancel() + } +} diff --git a/builder/openstack-new/builder_test.go b/builder/openstack-new/builder_test.go new file mode 100644 index 000000000..badf9784d --- /dev/null +++ b/builder/openstack-new/builder_test.go @@ -0,0 +1,94 @@ +package openstack + +import ( + "github.com/mitchellh/packer/packer" + "testing" +) + +func testConfig() map[string]interface{} { + return map[string]interface{}{ + "username": "foo", + "password": "bar", + "provider": "foo", + "region": "DFW", + "image_name": "foo", + "source_image": "foo", + "flavor": "foo", + "ssh_username": "root", + } +} + +func TestBuilder_ImplementsBuilder(t *testing.T) { + var raw interface{} + raw = &Builder{} + if _, ok := raw.(packer.Builder); !ok { + t.Fatalf("Builder should be a builder") + } +} + +func TestBuilder_Prepare_BadType(t *testing.T) { + b := &Builder{} + c := map[string]interface{}{ + "password": []string{}, + } + + warns, err := b.Prepare(c) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err == nil { + t.Fatalf("prepare should fail") + } +} + +func TestBuilderPrepare_ImageName(t *testing.T) { + var b Builder + config := testConfig() + + // Test good + config["image_name"] = "foo" + warns, err := b.Prepare(config) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + // Test bad + config["image_name"] = "foo {{" + b = Builder{} + warns, err = b.Prepare(config) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err == nil { + t.Fatal("should have error") + } + + // Test bad + delete(config, "image_name") + b = Builder{} + warns, err = b.Prepare(config) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err == nil { + t.Fatal("should have error") + } +} + +func TestBuilderPrepare_InvalidKey(t *testing.T) { + var b Builder + config := testConfig() + + // Add a random key + config["i_should_not_be_valid"] = true + warns, err := b.Prepare(config) + if len(warns) > 0 { + t.Fatalf("bad: %#v", warns) + } + if err == nil { + t.Fatal("should have error") + } +} diff --git a/builder/openstack-new/image_config.go b/builder/openstack-new/image_config.go new file mode 100644 index 000000000..124449eab --- /dev/null +++ b/builder/openstack-new/image_config.go @@ -0,0 +1,25 @@ +package openstack + +import ( + "fmt" + + "github.com/mitchellh/packer/template/interpolate" +) + +// ImageConfig is for common configuration related to creating Images. +type ImageConfig struct { + ImageName string `mapstructure:"image_name"` +} + +func (c *ImageConfig) Prepare(ctx *interpolate.Context) []error { + errs := make([]error, 0) + if c.ImageName == "" { + errs = append(errs, fmt.Errorf("An image_name must be specified")) + } + + if len(errs) > 0 { + return errs + } + + return nil +} diff --git a/builder/openstack-new/image_config_test.go b/builder/openstack-new/image_config_test.go new file mode 100644 index 000000000..4d81ecd94 --- /dev/null +++ b/builder/openstack-new/image_config_test.go @@ -0,0 +1,23 @@ +package openstack + +import ( + "testing" +) + +func testImageConfig() *ImageConfig { + return &ImageConfig{ + ImageName: "foo", + } +} + +func TestImageConfigPrepare_Region(t *testing.T) { + c := testImageConfig() + if err := c.Prepare(nil); err != nil { + t.Fatalf("shouldn't have err: %s", err) + } + + c.ImageName = "" + if err := c.Prepare(nil); err == nil { + t.Fatal("should have error") + } +} diff --git a/builder/openstack-new/run_config.go b/builder/openstack-new/run_config.go new file mode 100644 index 000000000..e5d73c9c1 --- /dev/null +++ b/builder/openstack-new/run_config.go @@ -0,0 +1,75 @@ +package openstack + +import ( + "errors" + "fmt" + "time" + + "github.com/mitchellh/packer/template/interpolate" +) + +// RunConfig contains configuration for running an instance from a source +// image and details on how to access that launched image. +type RunConfig struct { + SourceImage string `mapstructure:"source_image"` + Flavor string `mapstructure:"flavor"` + RawSSHTimeout string `mapstructure:"ssh_timeout"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort int `mapstructure:"ssh_port"` + SSHInterface string `mapstructure:"ssh_interface"` + OpenstackProvider string `mapstructure:"openstack_provider"` + UseFloatingIp bool `mapstructure:"use_floating_ip"` + RackconnectWait bool `mapstructure:"rackconnect_wait"` + FloatingIpPool string `mapstructure:"floating_ip_pool"` + FloatingIp string `mapstructure:"floating_ip"` + SecurityGroups []string `mapstructure:"security_groups"` + Networks []string `mapstructure:"networks"` + + // Unexported fields that are calculated from others + sshTimeout time.Duration +} + +func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { + // Defaults + if c.SSHUsername == "" { + c.SSHUsername = "root" + } + + if c.SSHPort == 0 { + c.SSHPort = 22 + } + + if c.RawSSHTimeout == "" { + c.RawSSHTimeout = "5m" + } + + if c.UseFloatingIp && c.FloatingIpPool == "" { + c.FloatingIpPool = "public" + } + + // Validation + var err error + errs := make([]error, 0) + if c.SourceImage == "" { + errs = append(errs, errors.New("A source_image must be specified")) + } + + if c.Flavor == "" { + errs = append(errs, errors.New("A flavor must be specified")) + } + + if c.SSHUsername == "" { + errs = append(errs, errors.New("An ssh_username must be specified")) + } + + c.sshTimeout, err = time.ParseDuration(c.RawSSHTimeout) + if err != nil { + errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) + } + + return errs +} + +func (c *RunConfig) SSHTimeout() time.Duration { + return c.sshTimeout +} diff --git a/builder/openstack-new/run_config_test.go b/builder/openstack-new/run_config_test.go new file mode 100644 index 000000000..16b89b352 --- /dev/null +++ b/builder/openstack-new/run_config_test.go @@ -0,0 +1,88 @@ +package openstack + +import ( + "os" + "testing" +) + +func init() { + // Clear out the openstack env vars so they don't + // affect our tests. + os.Setenv("SDK_USERNAME", "") + os.Setenv("SDK_PASSWORD", "") + os.Setenv("SDK_PROVIDER", "") +} + +func testRunConfig() *RunConfig { + return &RunConfig{ + SourceImage: "abcd", + Flavor: "m1.small", + SSHUsername: "root", + } +} + +func TestRunConfigPrepare(t *testing.T) { + c := testRunConfig() + err := c.Prepare(nil) + if len(err) > 0 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_InstanceType(t *testing.T) { + c := testRunConfig() + c.Flavor = "" + if err := c.Prepare(nil); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SourceImage(t *testing.T) { + c := testRunConfig() + c.SourceImage = "" + if err := c.Prepare(nil); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SSHPort(t *testing.T) { + c := testRunConfig() + c.SSHPort = 0 + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.SSHPort != 22 { + t.Fatalf("invalid value: %d", c.SSHPort) + } + + c.SSHPort = 44 + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + if c.SSHPort != 44 { + t.Fatalf("invalid value: %d", c.SSHPort) + } +} + +func TestRunConfigPrepare_SSHTimeout(t *testing.T) { + c := testRunConfig() + c.RawSSHTimeout = "" + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } + + c.RawSSHTimeout = "bad" + if err := c.Prepare(nil); len(err) != 1 { + t.Fatalf("err: %s", err) + } +} + +func TestRunConfigPrepare_SSHUsername(t *testing.T) { + c := testRunConfig() + c.SSHUsername = "" + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } +} diff --git a/builder/openstack-new/server.go b/builder/openstack-new/server.go new file mode 100644 index 000000000..a87ef0110 --- /dev/null +++ b/builder/openstack-new/server.go @@ -0,0 +1,100 @@ +package openstack + +import ( + "errors" + "fmt" + "log" + "time" + + "github.com/mitchellh/multistep" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +// StateRefreshFunc is a function type used for StateChangeConf that is +// responsible for refreshing the item being watched for a state change. +// +// It returns three results. `result` is any object that will be returned +// as the final object after waiting for state change. This allows you to +// return the final updated object, for example an openstack instance after +// refreshing it. +// +// `state` is the latest state of that object. And `err` is any error that +// may have happened while refreshing the state. +type StateRefreshFunc func() (result interface{}, state string, progress int, err error) + +// StateChangeConf is the configuration struct used for `WaitForState`. +type StateChangeConf struct { + Pending []string + Refresh StateRefreshFunc + StepState multistep.StateBag + Target string +} + +// ServerStateRefreshFunc returns a StateRefreshFunc that is used to watch +// an openstack server. +func ServerStateRefreshFunc( + client *gophercloud.ServiceClient, s *servers.Server) StateRefreshFunc { + return func() (interface{}, string, int, error) { + var serverNew *servers.Server + result := servers.Get(client, s.ID) + err := result.Err + if err == nil { + serverNew, err = result.Extract() + } + if result.Err != nil { + errCode, ok := result.Err.(*gophercloud.UnexpectedResponseCodeError) + if ok && errCode.Actual == 404 { + log.Printf("[INFO] 404 on ServerStateRefresh, returning DELETED") + return nil, "DELETED", 0, nil + } else { + log.Printf("[ERROR] Error on ServerStateRefresh: %s", result.Err) + return nil, "", 0, result.Err + } + } + + return serverNew, serverNew.Status, serverNew.Progress, nil + } +} + +// WaitForState watches an object and waits for it to achieve a certain +// state. +func WaitForState(conf *StateChangeConf) (i interface{}, err error) { + log.Printf("Waiting for state to become: %s", conf.Target) + + for { + var currentProgress int + var currentState string + i, currentState, currentProgress, err = conf.Refresh() + if err != nil { + return + } + + if currentState == conf.Target { + return + } + + if conf.StepState != nil { + if _, ok := conf.StepState.GetOk(multistep.StateCancelled); ok { + return nil, errors.New("interrupted") + } + } + + found := false + for _, allowed := range conf.Pending { + if currentState == allowed { + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target) + } + + log.Printf("Waiting for state to become: %s currently %s (%d%%)", conf.Target, currentState, currentProgress) + time.Sleep(2 * time.Second) + } + + return +} diff --git a/builder/openstack-new/ssh.go b/builder/openstack-new/ssh.go new file mode 100644 index 000000000..a3de654f6 --- /dev/null +++ b/builder/openstack-new/ssh.go @@ -0,0 +1,89 @@ +package openstack + +import ( + "errors" + "fmt" + "time" + + "github.com/mitchellh/multistep" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" + "golang.org/x/crypto/ssh" +) + +// SSHAddress returns a function that can be given to the SSH communicator +// for determining the SSH address based on the server AccessIPv4 setting.. +func SSHAddress( + client *gophercloud.ServiceClient, + sshinterface string, port int) func(multistep.StateBag) (string, error) { + return func(state multistep.StateBag) (string, error) { + s := state.Get("server").(*servers.Server) + + // If we have a floating IP, use that + if ip := state.Get("access_ip").(*floatingip.FloatingIP); ip.FixedIP != "" { + return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil + } + + if s.AccessIPv4 != "" { + return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil + } + + // Get all the addresses associated with this server + /* + ip_pools, err := s.AllAddressPools() + if err != nil { + return "", errors.New("Error parsing SSH addresses") + } + for pool, addresses := range ip_pools { + if sshinterface != "" { + if pool != sshinterface { + continue + } + } + if pool != "" { + for _, address := range addresses { + if address.Addr != "" && address.Version == 4 { + return fmt.Sprintf("%s:%d", address.Addr, port), nil + } + } + } + } + */ + + result := servers.Get(client, s.ID) + err := result.Err + if err == nil { + s, err = result.Extract() + } + if err != nil { + return "", err + } + + state.Put("server", s) + time.Sleep(1 * time.Second) + + return "", errors.New("couldn't determine IP address for server") + } +} + +// SSHConfig returns a function that can be used for the SSH communicator +// config for connecting to the instance created over SSH using the generated +// private key. +func SSHConfig(username string) func(multistep.StateBag) (*ssh.ClientConfig, error) { + return func(state multistep.StateBag) (*ssh.ClientConfig, error) { + privateKey := state.Get("privateKey").(string) + + signer, err := ssh.ParsePrivateKey([]byte(privateKey)) + if err != nil { + return nil, fmt.Errorf("Error setting up SSH config: %s", err) + } + + return &ssh.ClientConfig{ + User: username, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + }, nil + } +} diff --git a/builder/openstack-new/step_allocate_ip.go b/builder/openstack-new/step_allocate_ip.go new file mode 100644 index 000000000..adb15eb5b --- /dev/null +++ b/builder/openstack-new/step_allocate_ip.go @@ -0,0 +1,94 @@ +package openstack + +import ( + "fmt" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +type StepAllocateIp struct { + FloatingIpPool string + FloatingIp string +} + +func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { + ui := state.Get("ui").(packer.Ui) + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + var instanceIp *floatingip.FloatingIP + // This is here in case we error out before putting instanceIp into the + // statebag below, because it is requested by Cleanup() + state.Put("access_ip", instanceIp) + + if s.FloatingIp != "" { + instanceIp.FixedIP = s.FloatingIp + } else if s.FloatingIpPool != "" { + newIp, err := floatingip.Create(client, floatingip.CreateOpts{ + Pool: s.FloatingIpPool, + }).Extract() + if err != nil { + err := fmt.Errorf("Error creating floating ip from pool '%s'", s.FloatingIpPool) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + instanceIp = newIp + ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) + } + + if instanceIp.FixedIP != "" { + err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() + if err != nil { + err := fmt.Errorf( + "Error associating floating IP %s with instance.", + instanceIp.FixedIP) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + ui.Say(fmt.Sprintf( + "Added floating IP %s to instance...", instanceIp.FixedIP)) + } + + state.Put("access_ip", instanceIp) + + return multistep.ActionContinue +} + +func (s *StepAllocateIp) Cleanup(state multistep.StateBag) { + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + instanceIp := state.Get("access_ip").(*floatingip.FloatingIP) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf( + "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + return + } + + if s.FloatingIpPool != "" && instanceIp.ID != "" { + if err := floatingip.Delete(client, instanceIp.ID).ExtractErr(); err != nil { + ui.Error(fmt.Sprintf( + "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + return + } + + ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.FixedIP)) + } +} diff --git a/builder/openstack-new/step_create_image.go b/builder/openstack-new/step_create_image.go new file mode 100644 index 000000000..df540e311 --- /dev/null +++ b/builder/openstack-new/step_create_image.go @@ -0,0 +1,82 @@ +package openstack + +import ( + "fmt" + "log" + "time" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/images" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +type stepCreateImage struct{} + +func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + // Create the image + ui.Say(fmt.Sprintf("Creating the image: %s", config.ImageName)) + imageId, err := servers.CreateImage(client, server.ID, servers.CreateImageOpts{ + Name: config.ImageName, + }).ExtractImageID() + if err != nil { + err := fmt.Errorf("Error creating image: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Set the Image ID in the state + ui.Say(fmt.Sprintf("Image: %s", imageId)) + state.Put("image", imageId) + + // Wait for the image to become ready + ui.Say("Waiting for image to become ready...") + if err := WaitForImage(client, imageId); err != nil { + err := fmt.Errorf("Error waiting for image: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *stepCreateImage) Cleanup(multistep.StateBag) { + // No cleanup... +} + +// WaitForImage waits for the given Image ID to become ready. +func WaitForImage(client *gophercloud.ServiceClient, imageId string) error { + for { + var image *images.Image + result := images.Get(client, imageId) + err := result.Err + if err == nil { + image, err = result.Extract() + } + if err != nil { + return err + } + + if image.Status == "ACTIVE" { + return nil + } + + log.Printf("Waiting for image creation status: %s (%d%%)", image.Status, image.Progress) + time.Sleep(2 * time.Second) + } +} diff --git a/builder/openstack-new/step_key_pair.go b/builder/openstack-new/step_key_pair.go new file mode 100644 index 000000000..06bcbf9ea --- /dev/null +++ b/builder/openstack-new/step_key_pair.go @@ -0,0 +1,106 @@ +package openstack + +import ( + "fmt" + "os" + "runtime" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" +) + +type StepKeyPair struct { + Debug bool + DebugKeyPath string + keyName string +} + +func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Say("Creating temporary keypair for this instance...") + keyName := fmt.Sprintf("packer %s", uuid.TimeOrderedUUID()) + keypair, err := keypairs.Create(computeClient, keypairs.CreateOpts{ + Name: keyName, + }).Extract() + if err != nil { + state.Put("error", fmt.Errorf("Error creating temporary keypair: %s", err)) + return multistep.ActionHalt + } + + if keypair.PrivateKey == "" { + state.Put("error", fmt.Errorf("The temporary keypair returned was blank")) + return multistep.ActionHalt + } + + // If we're in debug mode, output the private key to the working + // directory. + if s.Debug { + ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath)) + f, err := os.Create(s.DebugKeyPath) + if err != nil { + state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) + return multistep.ActionHalt + } + defer f.Close() + + // Write the key out + if _, err := f.Write([]byte(keypair.PrivateKey)); err != nil { + state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) + return multistep.ActionHalt + } + + // Chmod it so that it is SSH ready + if runtime.GOOS != "windows" { + if err := f.Chmod(0600); err != nil { + state.Put("error", fmt.Errorf("Error setting permissions of debug key: %s", err)) + return multistep.ActionHalt + } + } + } + + // Set the keyname so we know to delete it later + s.keyName = keyName + + // Set some state data for use in future steps + state.Put("keyPair", keyName) + state.Put("privateKey", keypair.PrivateKey) + + return multistep.ActionContinue +} + +func (s *StepKeyPair) Cleanup(state multistep.StateBag) { + // If no key name is set, then we never created it, so just return + if s.keyName == "" { + return + } + + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf( + "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) + return + } + + ui.Say("Deleting temporary keypair...") + err = keypairs.Delete(computeClient, s.keyName).ExtractErr() + if err != nil { + ui.Error(fmt.Sprintf( + "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) + } +} diff --git a/builder/openstack-new/step_run_source_server.go b/builder/openstack-new/step_run_source_server.go new file mode 100644 index 000000000..e58e2c46b --- /dev/null +++ b/builder/openstack-new/step_run_source_server.go @@ -0,0 +1,110 @@ +package openstack + +import ( + "fmt" + "log" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +type StepRunSourceServer struct { + Flavor string + Name string + SourceImage string + SecurityGroups []string + Networks []string + + server *servers.Server +} + +func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(Config) + keyName := state.Get("keyPair").(string) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + networks := make([]servers.Network, len(s.Networks)) + for i, networkUuid := range s.Networks { + networks[i].UUID = networkUuid + } + + s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ + CreateOptsBuilder: servers.CreateOpts{ + Name: s.Name, + ImageRef: s.SourceImage, + FlavorRef: s.Flavor, + SecurityGroups: s.SecurityGroups, + Networks: networks, + }, + + KeyName: keyName, + }).Extract() + if err != nil { + err := fmt.Errorf("Error launching source server: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + log.Printf("server id: %s", s.server.ID) + + ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.ID)) + stateChange := StateChangeConf{ + Pending: []string{"BUILD"}, + Target: "ACTIVE", + Refresh: ServerStateRefreshFunc(computeClient, s.server), + StepState: state, + } + latestServer, err := WaitForState(&stateChange) + if err != nil { + err := fmt.Errorf("Error waiting for server (%s) to become ready: %s", s.server.ID, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + s.server = latestServer.(*servers.Server) + state.Put("server", s.server) + + return multistep.ActionContinue +} + +func (s *StepRunSourceServer) Cleanup(state multistep.StateBag) { + if s.server == nil { + return + } + + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) + return + } + + ui.Say("Terminating the source server...") + if err := servers.Delete(computeClient, s.server.ID).ExtractErr(); err != nil { + ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) + return + } + + stateChange := StateChangeConf{ + Pending: []string{"ACTIVE", "BUILD", "REBUILD", "SUSPENDED"}, + Refresh: ServerStateRefreshFunc(computeClient, s.server), + Target: "DELETED", + } + + WaitForState(&stateChange) +} diff --git a/builder/openstack-new/step_wait_for_rackconnect.go b/builder/openstack-new/step_wait_for_rackconnect.go new file mode 100644 index 000000000..6263bd17d --- /dev/null +++ b/builder/openstack-new/step_wait_for_rackconnect.go @@ -0,0 +1,52 @@ +package openstack + +import ( + "fmt" + "time" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" +) + +type StepWaitForRackConnect struct { + Wait bool +} + +func (s *StepWaitForRackConnect) Run(state multistep.StateBag) multistep.StepAction { + if !s.Wait { + return multistep.ActionContinue + } + + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Say(fmt.Sprintf( + "Waiting for server (%s) to become RackConnect ready...", server.ID)) + for { + server, err = servers.Get(computeClient, server.ID).Extract() + if err != nil { + return multistep.ActionHalt + } + + if server.Metadata["rackconnect_automation_status"] == "DEPLOYED" { + break + } + + time.Sleep(2 * time.Second) + } + + return multistep.ActionContinue +} + +func (s *StepWaitForRackConnect) Cleanup(state multistep.StateBag) { +} diff --git a/plugin/builder-openstack-new/main.go b/plugin/builder-openstack-new/main.go new file mode 100644 index 000000000..d8075c78d --- /dev/null +++ b/plugin/builder-openstack-new/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "github.com/mitchellh/packer/builder/openstack-new" + "github.com/mitchellh/packer/packer/plugin" +) + +func main() { + server, err := plugin.Server() + if err != nil { + panic(err) + } + server.RegisterBuilder(new(openstack.Builder)) + server.Serve() +} From 551e80774d0b0f28547a5373a59821fbf0cfeaf8 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:03:17 -0400 Subject: [PATCH 67/85] builder/openstack-new: fix some issues --- builder/openstack-new/ssh.go | 53 ++++++++++--------- builder/openstack-new/step_allocate_ip.go | 7 ++- .../openstack-new/step_run_source_server.go | 2 +- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/builder/openstack-new/ssh.go b/builder/openstack-new/ssh.go index a3de654f6..7b0510f98 100644 --- a/builder/openstack-new/ssh.go +++ b/builder/openstack-new/ssh.go @@ -3,6 +3,7 @@ package openstack import ( "errors" "fmt" + "log" "time" "github.com/mitchellh/multistep" @@ -21,7 +22,8 @@ func SSHAddress( s := state.Get("server").(*servers.Server) // If we have a floating IP, use that - if ip := state.Get("access_ip").(*floatingip.FloatingIP); ip.FixedIP != "" { + ip := state.Get("access_ip").(*floatingip.FloatingIP) + if ip != nil && ip.FixedIP != "" { return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil } @@ -29,33 +31,34 @@ func SSHAddress( return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil } - // Get all the addresses associated with this server - /* - ip_pools, err := s.AllAddressPools() - if err != nil { - return "", errors.New("Error parsing SSH addresses") + // Get all the addresses associated with this server. This + // was taken directly from Terraform. + for _, networkAddresses := range s.Addresses { + elements, ok := networkAddresses.([]interface{}) + if !ok { + log.Printf( + "[ERROR] Unknown return type for address field: %#v", + networkAddresses) + continue } - for pool, addresses := range ip_pools { - if sshinterface != "" { - if pool != sshinterface { - continue - } - } - if pool != "" { - for _, address := range addresses { - if address.Addr != "" && address.Version == 4 { - return fmt.Sprintf("%s:%d", address.Addr, port), nil - } - } - } - } - */ - result := servers.Get(client, s.ID) - err := result.Err - if err == nil { - s, err = result.Extract() + for _, element := range elements { + var addr string + address := element.(map[string]interface{}) + if address["OS-EXT-IPS:type"] == "floating" { + addr = address["addr"].(string) + } else { + if address["version"].(float64) == 4 { + addr = address["addr"].(string) + } + } + if addr != "" { + return fmt.Sprintf("%s:%d", addr, port), nil + } + } } + + s, err := servers.Get(client, s.ID).Extract() if err != nil { return "", err } diff --git a/builder/openstack-new/step_allocate_ip.go b/builder/openstack-new/step_allocate_ip.go index adb15eb5b..16efe8d38 100644 --- a/builder/openstack-new/step_allocate_ip.go +++ b/builder/openstack-new/step_allocate_ip.go @@ -33,7 +33,7 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { state.Put("access_ip", instanceIp) if s.FloatingIp != "" { - instanceIp.FixedIP = s.FloatingIp + *instanceIp = floatingip.FloatingIP{FixedIP: s.FloatingIp} } else if s.FloatingIpPool != "" { newIp, err := floatingip.Create(client, floatingip.CreateOpts{ Pool: s.FloatingIpPool, @@ -45,11 +45,11 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - instanceIp = newIp + *instanceIp = *newIp ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) } - if instanceIp.FixedIP != "" { + if instanceIp != nil && instanceIp.FixedIP != "" { err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() if err != nil { err := fmt.Errorf( @@ -65,7 +65,6 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { } state.Put("access_ip", instanceIp) - return multistep.ActionContinue } diff --git a/builder/openstack-new/step_run_source_server.go b/builder/openstack-new/step_run_source_server.go index e58e2c46b..4432d5860 100644 --- a/builder/openstack-new/step_run_source_server.go +++ b/builder/openstack-new/step_run_source_server.go @@ -42,7 +42,7 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction CreateOptsBuilder: servers.CreateOpts{ Name: s.Name, ImageRef: s.SourceImage, - FlavorRef: s.Flavor, + FlavorName: s.Flavor, SecurityGroups: s.SecurityGroups, Networks: networks, }, From 7a46b80cfb24aa307286a2b67c253f7716b3d3fc Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:05:24 -0400 Subject: [PATCH 68/85] builder/openstack-new: better UI --- builder/openstack-new/step_create_image.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/openstack-new/step_create_image.go b/builder/openstack-new/step_create_image.go index df540e311..989db81e5 100644 --- a/builder/openstack-new/step_create_image.go +++ b/builder/openstack-new/step_create_image.go @@ -40,7 +40,7 @@ func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { } // Set the Image ID in the state - ui.Say(fmt.Sprintf("Image: %s", imageId)) + ui.Message(fmt.Sprintf("Image: %s", imageId)) state.Put("image", imageId) // Wait for the image to become ready From 46f518f21df6a127d3e9e64eabfdc31441aa4a1c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:09:01 -0400 Subject: [PATCH 69/85] builder/openstack: proper error extraction --- builder/openstack-new/server.go | 15 +++++---------- builder/openstack-new/step_create_image.go | 14 ++++++++------ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/builder/openstack-new/server.go b/builder/openstack-new/server.go index a87ef0110..de8c9d103 100644 --- a/builder/openstack-new/server.go +++ b/builder/openstack-new/server.go @@ -36,20 +36,15 @@ type StateChangeConf struct { func ServerStateRefreshFunc( client *gophercloud.ServiceClient, s *servers.Server) StateRefreshFunc { return func() (interface{}, string, int, error) { - var serverNew *servers.Server - result := servers.Get(client, s.ID) - err := result.Err - if err == nil { - serverNew, err = result.Extract() - } - if result.Err != nil { - errCode, ok := result.Err.(*gophercloud.UnexpectedResponseCodeError) + serverNew, err := servers.Get(client, s.ID).Extract() + if err != nil { + errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) if ok && errCode.Actual == 404 { log.Printf("[INFO] 404 on ServerStateRefresh, returning DELETED") return nil, "DELETED", 0, nil } else { - log.Printf("[ERROR] Error on ServerStateRefresh: %s", result.Err) - return nil, "", 0, result.Err + log.Printf("[ERROR] Error on ServerStateRefresh: %s", err) + return nil, "", 0, err } } diff --git a/builder/openstack-new/step_create_image.go b/builder/openstack-new/step_create_image.go index 989db81e5..b777e8b0b 100644 --- a/builder/openstack-new/step_create_image.go +++ b/builder/openstack-new/step_create_image.go @@ -62,13 +62,15 @@ func (s *stepCreateImage) Cleanup(multistep.StateBag) { // WaitForImage waits for the given Image ID to become ready. func WaitForImage(client *gophercloud.ServiceClient, imageId string) error { for { - var image *images.Image - result := images.Get(client, imageId) - err := result.Err - if err == nil { - image, err = result.Extract() - } + image, err := images.Get(client, imageId).Extract() if err != nil { + errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) + if ok && errCode.Actual == 500 { + log.Printf("[ERROR] 500 error received, will ignore and retry: %s", err) + time.Sleep(2 * time.Second) + continue + } + return err } From a0d41fcd14f2525df351d1764e3401d1f016ff53 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:16:43 -0400 Subject: [PATCH 70/85] builder/openstack --- builder/openstack-new/access_config.go | 109 ------------ builder/openstack-new/artifact.go | 47 ----- builder/openstack-new/artifact_test.go | 35 ---- builder/openstack-new/builder.go | 134 -------------- builder/openstack-new/builder_test.go | 94 ---------- builder/openstack-new/image_config.go | 25 --- builder/openstack-new/image_config_test.go | 23 --- builder/openstack-new/run_config.go | 75 -------- builder/openstack-new/run_config_test.go | 88 ---------- builder/openstack-new/server.go | 95 ---------- builder/openstack-new/ssh.go | 92 ---------- builder/openstack-new/step_allocate_ip.go | 93 ---------- builder/openstack-new/step_create_image.go | 84 --------- builder/openstack-new/step_key_pair.go | 106 ------------ .../openstack-new/step_run_source_server.go | 110 ------------ .../step_wait_for_rackconnect.go | 52 ------ builder/openstack/access_config.go | 163 +++++++++--------- builder/openstack/access_config_test.go | 77 --------- builder/openstack/artifact.go | 7 +- builder/openstack/builder.go | 23 +-- builder/openstack/server.go | 23 +-- builder/openstack/ssh.go | 62 ++++--- builder/openstack/step_allocate_ip.go | 68 +++++--- builder/openstack/step_create_image.go | 39 +++-- builder/openstack/step_key_pair.go | 43 +++-- builder/openstack/step_run_source_server.go | 70 ++++---- .../openstack/step_wait_for_rackconnect.go | 22 ++- 27 files changed, 303 insertions(+), 1556 deletions(-) delete mode 100644 builder/openstack-new/access_config.go delete mode 100644 builder/openstack-new/artifact.go delete mode 100644 builder/openstack-new/artifact_test.go delete mode 100644 builder/openstack-new/builder.go delete mode 100644 builder/openstack-new/builder_test.go delete mode 100644 builder/openstack-new/image_config.go delete mode 100644 builder/openstack-new/image_config_test.go delete mode 100644 builder/openstack-new/run_config.go delete mode 100644 builder/openstack-new/run_config_test.go delete mode 100644 builder/openstack-new/server.go delete mode 100644 builder/openstack-new/ssh.go delete mode 100644 builder/openstack-new/step_allocate_ip.go delete mode 100644 builder/openstack-new/step_create_image.go delete mode 100644 builder/openstack-new/step_key_pair.go delete mode 100644 builder/openstack-new/step_run_source_server.go delete mode 100644 builder/openstack-new/step_wait_for_rackconnect.go delete mode 100644 builder/openstack/access_config_test.go diff --git a/builder/openstack-new/access_config.go b/builder/openstack-new/access_config.go deleted file mode 100644 index e0f962c50..000000000 --- a/builder/openstack-new/access_config.go +++ /dev/null @@ -1,109 +0,0 @@ -package openstack - -import ( - "crypto/tls" - "fmt" - "net/http" - "os" - - "github.com/mitchellh/packer/template/interpolate" - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack" -) - -// AccessConfig is for common configuration related to openstack access -type AccessConfig struct { - Username string `mapstructure:"username"` - UserID string `mapstructure:"user_id"` - Password string `mapstructure:"password"` - APIKey string `mapstructure:"api_key"` - IdentityEndpoint string `mapstructure:"identity_endpoint"` - TenantID string `mapstructure:"tenant_id"` - TenantName string `mapstructure:"tenant_name"` - DomainID string `mapstructure:"domain_id"` - DomainName string `mapstructure:"domain_name"` - Insecure bool `mapstructure:"insecure"` - Region string `mapstructure:"region"` - EndpointType string `mapstructure:"endpoint_type"` - - osClient *gophercloud.ProviderClient -} - -func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { - if c.EndpointType != "internal" && c.EndpointType != "internalURL" && - c.EndpointType != "admin" && c.EndpointType != "adminURL" && - c.EndpointType != "public" && c.EndpointType != "publicURL" && - c.EndpointType != "" { - return []error{fmt.Errorf("Invalid endpoint type provided")} - } - - if c.Region == "" { - c.Region = os.Getenv("OS_REGION_NAME") - } - - // Get as much as possible from the end - ao, err := openstack.AuthOptionsFromEnv() - if err != nil { - return []error{err} - } - - // Override values if we have them in our config - overrides := []struct { - From, To *string - }{ - {&c.Username, &ao.Username}, - {&c.UserID, &ao.UserID}, - {&c.Password, &ao.Password}, - {&c.APIKey, &ao.APIKey}, - {&c.IdentityEndpoint, &ao.IdentityEndpoint}, - {&c.TenantID, &ao.TenantID}, - {&c.TenantName, &ao.TenantName}, - {&c.DomainID, &ao.DomainID}, - {&c.DomainName, &ao.DomainName}, - } - for _, s := range overrides { - if *s.From != "" { - *s.To = *s.From - } - } - - // Build the client itself - client, err := openstack.NewClient(ao.IdentityEndpoint) - if err != nil { - return []error{err} - } - - // If we have insecure set, then create a custom HTTP client that - // ignores SSL errors. - if c.Insecure { - config := &tls.Config{InsecureSkipVerify: true} - transport := &http.Transport{TLSClientConfig: config} - client.HTTPClient.Transport = transport - } - - // Auth - err = openstack.Authenticate(client, ao) - if err != nil { - return []error{err} - } - - c.osClient = client - return nil -} - -func (c *AccessConfig) computeV2Client() (*gophercloud.ServiceClient, error) { - return openstack.NewComputeV2(c.osClient, gophercloud.EndpointOpts{ - Region: c.Region, - Availability: c.getEndpointType(), - }) -} - -func (c *AccessConfig) getEndpointType() gophercloud.Availability { - if c.EndpointType == "internal" || c.EndpointType == "internalURL" { - return gophercloud.AvailabilityInternal - } - if c.EndpointType == "admin" || c.EndpointType == "adminURL" { - return gophercloud.AvailabilityAdmin - } - return gophercloud.AvailabilityPublic -} diff --git a/builder/openstack-new/artifact.go b/builder/openstack-new/artifact.go deleted file mode 100644 index aa60d2641..000000000 --- a/builder/openstack-new/artifact.go +++ /dev/null @@ -1,47 +0,0 @@ -package openstack - -import ( - "fmt" - "log" - - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack/compute/v2/images" -) - -// Artifact is an artifact implementation that contains built images. -type Artifact struct { - // ImageId of built image - ImageId string - - // BuilderId is the unique ID for the builder that created this image - BuilderIdValue string - - // OpenStack connection for performing API stuff. - Client *gophercloud.ServiceClient -} - -func (a *Artifact) BuilderId() string { - return a.BuilderIdValue -} - -func (*Artifact) Files() []string { - // We have no files - return nil -} - -func (a *Artifact) Id() string { - return a.ImageId -} - -func (a *Artifact) String() string { - return fmt.Sprintf("An image was created: %v", a.ImageId) -} - -func (a *Artifact) State(name string) interface{} { - return nil -} - -func (a *Artifact) Destroy() error { - log.Printf("Destroying image: %s", a.ImageId) - return images.Delete(a.Client, a.ImageId).ExtractErr() -} diff --git a/builder/openstack-new/artifact_test.go b/builder/openstack-new/artifact_test.go deleted file mode 100644 index 313fea7cf..000000000 --- a/builder/openstack-new/artifact_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package openstack - -import ( - "github.com/mitchellh/packer/packer" - "testing" -) - -func TestArtifact_Impl(t *testing.T) { - var _ packer.Artifact = new(Artifact) -} - -func TestArtifactId(t *testing.T) { - expected := `b8cdf55b-c916-40bd-b190-389ec144c4ed` - - a := &Artifact{ - ImageId: "b8cdf55b-c916-40bd-b190-389ec144c4ed", - } - - result := a.Id() - if result != expected { - t.Fatalf("bad: %s", result) - } -} - -func TestArtifactString(t *testing.T) { - expected := "An image was created: b8cdf55b-c916-40bd-b190-389ec144c4ed" - - a := &Artifact{ - ImageId: "b8cdf55b-c916-40bd-b190-389ec144c4ed", - } - result := a.String() - if result != expected { - t.Fatalf("bad: %s", result) - } -} diff --git a/builder/openstack-new/builder.go b/builder/openstack-new/builder.go deleted file mode 100644 index bebb28452..000000000 --- a/builder/openstack-new/builder.go +++ /dev/null @@ -1,134 +0,0 @@ -// The openstack package contains a packer.Builder implementation that -// builds Images for openstack. - -package openstack - -import ( - "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/common" - "log" - - "github.com/mitchellh/packer/helper/config" - "github.com/mitchellh/packer/packer" - "github.com/mitchellh/packer/template/interpolate" -) - -// The unique ID for this builder -const BuilderId = "mitchellh.openstack" - -type Config struct { - common.PackerConfig `mapstructure:",squash"` - AccessConfig `mapstructure:",squash"` - ImageConfig `mapstructure:",squash"` - RunConfig `mapstructure:",squash"` - - ctx interpolate.Context -} - -type Builder struct { - config Config - runner multistep.Runner -} - -func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { - err := config.Decode(&b.config, &config.DecodeOpts{ - Interpolate: true, - }, raws...) - if err != nil { - return nil, err - } - - // Accumulate any errors - var errs *packer.MultiError - errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare(&b.config.ctx)...) - errs = packer.MultiErrorAppend(errs, b.config.ImageConfig.Prepare(&b.config.ctx)...) - errs = packer.MultiErrorAppend(errs, b.config.RunConfig.Prepare(&b.config.ctx)...) - - if errs != nil && len(errs.Errors) > 0 { - return nil, errs - } - - log.Println(common.ScrubConfig(b.config, b.config.Password)) - return nil, nil -} - -func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { - computeClient, err := b.config.computeV2Client() - if err != nil { - return nil, fmt.Errorf("Error initializing compute client: %s", err) - } - - // Setup the state bag and initial state for the steps - state := new(multistep.BasicStateBag) - state.Put("config", b.config) - state.Put("hook", hook) - state.Put("ui", ui) - - // Build the steps - steps := []multistep.Step{ - &StepKeyPair{ - Debug: b.config.PackerDebug, - DebugKeyPath: fmt.Sprintf("os_%s.pem", b.config.PackerBuildName), - }, - &StepRunSourceServer{ - Name: b.config.ImageName, - Flavor: b.config.Flavor, - SourceImage: b.config.SourceImage, - SecurityGroups: b.config.SecurityGroups, - Networks: b.config.Networks, - }, - &StepWaitForRackConnect{ - Wait: b.config.RackconnectWait, - }, - &StepAllocateIp{ - FloatingIpPool: b.config.FloatingIpPool, - FloatingIp: b.config.FloatingIp, - }, - &common.StepConnectSSH{ - SSHAddress: SSHAddress(computeClient, b.config.SSHInterface, b.config.SSHPort), - SSHConfig: SSHConfig(b.config.SSHUsername), - SSHWaitTimeout: b.config.SSHTimeout(), - }, - &common.StepProvision{}, - &stepCreateImage{}, - } - - // Run! - if b.config.PackerDebug { - b.runner = &multistep.DebugRunner{ - Steps: steps, - PauseFn: common.MultistepDebugFn(ui), - } - } else { - b.runner = &multistep.BasicRunner{Steps: steps} - } - - b.runner.Run(state) - - // If there was an error, return that - if rawErr, ok := state.GetOk("error"); ok { - return nil, rawErr.(error) - } - - // If there are no images, then just return - if _, ok := state.GetOk("image"); !ok { - return nil, nil - } - - // Build the artifact and return it - artifact := &Artifact{ - ImageId: state.Get("image").(string), - BuilderIdValue: BuilderId, - Client: computeClient, - } - - return artifact, nil -} - -func (b *Builder) Cancel() { - if b.runner != nil { - log.Println("Cancelling the step runner...") - b.runner.Cancel() - } -} diff --git a/builder/openstack-new/builder_test.go b/builder/openstack-new/builder_test.go deleted file mode 100644 index badf9784d..000000000 --- a/builder/openstack-new/builder_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package openstack - -import ( - "github.com/mitchellh/packer/packer" - "testing" -) - -func testConfig() map[string]interface{} { - return map[string]interface{}{ - "username": "foo", - "password": "bar", - "provider": "foo", - "region": "DFW", - "image_name": "foo", - "source_image": "foo", - "flavor": "foo", - "ssh_username": "root", - } -} - -func TestBuilder_ImplementsBuilder(t *testing.T) { - var raw interface{} - raw = &Builder{} - if _, ok := raw.(packer.Builder); !ok { - t.Fatalf("Builder should be a builder") - } -} - -func TestBuilder_Prepare_BadType(t *testing.T) { - b := &Builder{} - c := map[string]interface{}{ - "password": []string{}, - } - - warns, err := b.Prepare(c) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatalf("prepare should fail") - } -} - -func TestBuilderPrepare_ImageName(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["image_name"] = "foo" - warns, err := b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - // Test bad - config["image_name"] = "foo {{" - b = Builder{} - warns, err = b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } - - // Test bad - delete(config, "image_name") - b = Builder{} - warns, err = b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } -} - -func TestBuilderPrepare_InvalidKey(t *testing.T) { - var b Builder - config := testConfig() - - // Add a random key - config["i_should_not_be_valid"] = true - warns, err := b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } -} diff --git a/builder/openstack-new/image_config.go b/builder/openstack-new/image_config.go deleted file mode 100644 index 124449eab..000000000 --- a/builder/openstack-new/image_config.go +++ /dev/null @@ -1,25 +0,0 @@ -package openstack - -import ( - "fmt" - - "github.com/mitchellh/packer/template/interpolate" -) - -// ImageConfig is for common configuration related to creating Images. -type ImageConfig struct { - ImageName string `mapstructure:"image_name"` -} - -func (c *ImageConfig) Prepare(ctx *interpolate.Context) []error { - errs := make([]error, 0) - if c.ImageName == "" { - errs = append(errs, fmt.Errorf("An image_name must be specified")) - } - - if len(errs) > 0 { - return errs - } - - return nil -} diff --git a/builder/openstack-new/image_config_test.go b/builder/openstack-new/image_config_test.go deleted file mode 100644 index 4d81ecd94..000000000 --- a/builder/openstack-new/image_config_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package openstack - -import ( - "testing" -) - -func testImageConfig() *ImageConfig { - return &ImageConfig{ - ImageName: "foo", - } -} - -func TestImageConfigPrepare_Region(t *testing.T) { - c := testImageConfig() - if err := c.Prepare(nil); err != nil { - t.Fatalf("shouldn't have err: %s", err) - } - - c.ImageName = "" - if err := c.Prepare(nil); err == nil { - t.Fatal("should have error") - } -} diff --git a/builder/openstack-new/run_config.go b/builder/openstack-new/run_config.go deleted file mode 100644 index e5d73c9c1..000000000 --- a/builder/openstack-new/run_config.go +++ /dev/null @@ -1,75 +0,0 @@ -package openstack - -import ( - "errors" - "fmt" - "time" - - "github.com/mitchellh/packer/template/interpolate" -) - -// RunConfig contains configuration for running an instance from a source -// image and details on how to access that launched image. -type RunConfig struct { - SourceImage string `mapstructure:"source_image"` - Flavor string `mapstructure:"flavor"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort int `mapstructure:"ssh_port"` - SSHInterface string `mapstructure:"ssh_interface"` - OpenstackProvider string `mapstructure:"openstack_provider"` - UseFloatingIp bool `mapstructure:"use_floating_ip"` - RackconnectWait bool `mapstructure:"rackconnect_wait"` - FloatingIpPool string `mapstructure:"floating_ip_pool"` - FloatingIp string `mapstructure:"floating_ip"` - SecurityGroups []string `mapstructure:"security_groups"` - Networks []string `mapstructure:"networks"` - - // Unexported fields that are calculated from others - sshTimeout time.Duration -} - -func (c *RunConfig) Prepare(ctx *interpolate.Context) []error { - // Defaults - if c.SSHUsername == "" { - c.SSHUsername = "root" - } - - if c.SSHPort == 0 { - c.SSHPort = 22 - } - - if c.RawSSHTimeout == "" { - c.RawSSHTimeout = "5m" - } - - if c.UseFloatingIp && c.FloatingIpPool == "" { - c.FloatingIpPool = "public" - } - - // Validation - var err error - errs := make([]error, 0) - if c.SourceImage == "" { - errs = append(errs, errors.New("A source_image must be specified")) - } - - if c.Flavor == "" { - errs = append(errs, errors.New("A flavor must be specified")) - } - - if c.SSHUsername == "" { - errs = append(errs, errors.New("An ssh_username must be specified")) - } - - c.sshTimeout, err = time.ParseDuration(c.RawSSHTimeout) - if err != nil { - errs = append(errs, fmt.Errorf("Failed parsing ssh_timeout: %s", err)) - } - - return errs -} - -func (c *RunConfig) SSHTimeout() time.Duration { - return c.sshTimeout -} diff --git a/builder/openstack-new/run_config_test.go b/builder/openstack-new/run_config_test.go deleted file mode 100644 index 16b89b352..000000000 --- a/builder/openstack-new/run_config_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package openstack - -import ( - "os" - "testing" -) - -func init() { - // Clear out the openstack env vars so they don't - // affect our tests. - os.Setenv("SDK_USERNAME", "") - os.Setenv("SDK_PASSWORD", "") - os.Setenv("SDK_PROVIDER", "") -} - -func testRunConfig() *RunConfig { - return &RunConfig{ - SourceImage: "abcd", - Flavor: "m1.small", - SSHUsername: "root", - } -} - -func TestRunConfigPrepare(t *testing.T) { - c := testRunConfig() - err := c.Prepare(nil) - if len(err) > 0 { - t.Fatalf("err: %s", err) - } -} - -func TestRunConfigPrepare_InstanceType(t *testing.T) { - c := testRunConfig() - c.Flavor = "" - if err := c.Prepare(nil); len(err) != 1 { - t.Fatalf("err: %s", err) - } -} - -func TestRunConfigPrepare_SourceImage(t *testing.T) { - c := testRunConfig() - c.SourceImage = "" - if err := c.Prepare(nil); len(err) != 1 { - t.Fatalf("err: %s", err) - } -} - -func TestRunConfigPrepare_SSHPort(t *testing.T) { - c := testRunConfig() - c.SSHPort = 0 - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } - - if c.SSHPort != 22 { - t.Fatalf("invalid value: %d", c.SSHPort) - } - - c.SSHPort = 44 - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } - - if c.SSHPort != 44 { - t.Fatalf("invalid value: %d", c.SSHPort) - } -} - -func TestRunConfigPrepare_SSHTimeout(t *testing.T) { - c := testRunConfig() - c.RawSSHTimeout = "" - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } - - c.RawSSHTimeout = "bad" - if err := c.Prepare(nil); len(err) != 1 { - t.Fatalf("err: %s", err) - } -} - -func TestRunConfigPrepare_SSHUsername(t *testing.T) { - c := testRunConfig() - c.SSHUsername = "" - if err := c.Prepare(nil); len(err) != 0 { - t.Fatalf("err: %s", err) - } -} diff --git a/builder/openstack-new/server.go b/builder/openstack-new/server.go deleted file mode 100644 index de8c9d103..000000000 --- a/builder/openstack-new/server.go +++ /dev/null @@ -1,95 +0,0 @@ -package openstack - -import ( - "errors" - "fmt" - "log" - "time" - - "github.com/mitchellh/multistep" - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -// StateRefreshFunc is a function type used for StateChangeConf that is -// responsible for refreshing the item being watched for a state change. -// -// It returns three results. `result` is any object that will be returned -// as the final object after waiting for state change. This allows you to -// return the final updated object, for example an openstack instance after -// refreshing it. -// -// `state` is the latest state of that object. And `err` is any error that -// may have happened while refreshing the state. -type StateRefreshFunc func() (result interface{}, state string, progress int, err error) - -// StateChangeConf is the configuration struct used for `WaitForState`. -type StateChangeConf struct { - Pending []string - Refresh StateRefreshFunc - StepState multistep.StateBag - Target string -} - -// ServerStateRefreshFunc returns a StateRefreshFunc that is used to watch -// an openstack server. -func ServerStateRefreshFunc( - client *gophercloud.ServiceClient, s *servers.Server) StateRefreshFunc { - return func() (interface{}, string, int, error) { - serverNew, err := servers.Get(client, s.ID).Extract() - if err != nil { - errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) - if ok && errCode.Actual == 404 { - log.Printf("[INFO] 404 on ServerStateRefresh, returning DELETED") - return nil, "DELETED", 0, nil - } else { - log.Printf("[ERROR] Error on ServerStateRefresh: %s", err) - return nil, "", 0, err - } - } - - return serverNew, serverNew.Status, serverNew.Progress, nil - } -} - -// WaitForState watches an object and waits for it to achieve a certain -// state. -func WaitForState(conf *StateChangeConf) (i interface{}, err error) { - log.Printf("Waiting for state to become: %s", conf.Target) - - for { - var currentProgress int - var currentState string - i, currentState, currentProgress, err = conf.Refresh() - if err != nil { - return - } - - if currentState == conf.Target { - return - } - - if conf.StepState != nil { - if _, ok := conf.StepState.GetOk(multistep.StateCancelled); ok { - return nil, errors.New("interrupted") - } - } - - found := false - for _, allowed := range conf.Pending { - if currentState == allowed { - found = true - break - } - } - - if !found { - return nil, fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target) - } - - log.Printf("Waiting for state to become: %s currently %s (%d%%)", conf.Target, currentState, currentProgress) - time.Sleep(2 * time.Second) - } - - return -} diff --git a/builder/openstack-new/ssh.go b/builder/openstack-new/ssh.go deleted file mode 100644 index 7b0510f98..000000000 --- a/builder/openstack-new/ssh.go +++ /dev/null @@ -1,92 +0,0 @@ -package openstack - -import ( - "errors" - "fmt" - "log" - "time" - - "github.com/mitchellh/multistep" - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" - "golang.org/x/crypto/ssh" -) - -// SSHAddress returns a function that can be given to the SSH communicator -// for determining the SSH address based on the server AccessIPv4 setting.. -func SSHAddress( - client *gophercloud.ServiceClient, - sshinterface string, port int) func(multistep.StateBag) (string, error) { - return func(state multistep.StateBag) (string, error) { - s := state.Get("server").(*servers.Server) - - // If we have a floating IP, use that - ip := state.Get("access_ip").(*floatingip.FloatingIP) - if ip != nil && ip.FixedIP != "" { - return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil - } - - if s.AccessIPv4 != "" { - return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil - } - - // Get all the addresses associated with this server. This - // was taken directly from Terraform. - for _, networkAddresses := range s.Addresses { - elements, ok := networkAddresses.([]interface{}) - if !ok { - log.Printf( - "[ERROR] Unknown return type for address field: %#v", - networkAddresses) - continue - } - - for _, element := range elements { - var addr string - address := element.(map[string]interface{}) - if address["OS-EXT-IPS:type"] == "floating" { - addr = address["addr"].(string) - } else { - if address["version"].(float64) == 4 { - addr = address["addr"].(string) - } - } - if addr != "" { - return fmt.Sprintf("%s:%d", addr, port), nil - } - } - } - - s, err := servers.Get(client, s.ID).Extract() - if err != nil { - return "", err - } - - state.Put("server", s) - time.Sleep(1 * time.Second) - - return "", errors.New("couldn't determine IP address for server") - } -} - -// SSHConfig returns a function that can be used for the SSH communicator -// config for connecting to the instance created over SSH using the generated -// private key. -func SSHConfig(username string) func(multistep.StateBag) (*ssh.ClientConfig, error) { - return func(state multistep.StateBag) (*ssh.ClientConfig, error) { - privateKey := state.Get("privateKey").(string) - - signer, err := ssh.ParsePrivateKey([]byte(privateKey)) - if err != nil { - return nil, fmt.Errorf("Error setting up SSH config: %s", err) - } - - return &ssh.ClientConfig{ - User: username, - Auth: []ssh.AuthMethod{ - ssh.PublicKeys(signer), - }, - }, nil - } -} diff --git a/builder/openstack-new/step_allocate_ip.go b/builder/openstack-new/step_allocate_ip.go deleted file mode 100644 index 16efe8d38..000000000 --- a/builder/openstack-new/step_allocate_ip.go +++ /dev/null @@ -1,93 +0,0 @@ -package openstack - -import ( - "fmt" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -type StepAllocateIp struct { - FloatingIpPool string - FloatingIp string -} - -func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { - ui := state.Get("ui").(packer.Ui) - config := state.Get("config").(Config) - server := state.Get("server").(*servers.Server) - - // We need the v2 compute client - client, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - var instanceIp *floatingip.FloatingIP - // This is here in case we error out before putting instanceIp into the - // statebag below, because it is requested by Cleanup() - state.Put("access_ip", instanceIp) - - if s.FloatingIp != "" { - *instanceIp = floatingip.FloatingIP{FixedIP: s.FloatingIp} - } else if s.FloatingIpPool != "" { - newIp, err := floatingip.Create(client, floatingip.CreateOpts{ - Pool: s.FloatingIpPool, - }).Extract() - if err != nil { - err := fmt.Errorf("Error creating floating ip from pool '%s'", s.FloatingIpPool) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - *instanceIp = *newIp - ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) - } - - if instanceIp != nil && instanceIp.FixedIP != "" { - err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() - if err != nil { - err := fmt.Errorf( - "Error associating floating IP %s with instance.", - instanceIp.FixedIP) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - ui.Say(fmt.Sprintf( - "Added floating IP %s to instance...", instanceIp.FixedIP)) - } - - state.Put("access_ip", instanceIp) - return multistep.ActionContinue -} - -func (s *StepAllocateIp) Cleanup(state multistep.StateBag) { - config := state.Get("config").(Config) - ui := state.Get("ui").(packer.Ui) - instanceIp := state.Get("access_ip").(*floatingip.FloatingIP) - - // We need the v2 compute client - client, err := config.computeV2Client() - if err != nil { - ui.Error(fmt.Sprintf( - "Error deleting temporary floating IP %s", instanceIp.FixedIP)) - return - } - - if s.FloatingIpPool != "" && instanceIp.ID != "" { - if err := floatingip.Delete(client, instanceIp.ID).ExtractErr(); err != nil { - ui.Error(fmt.Sprintf( - "Error deleting temporary floating IP %s", instanceIp.FixedIP)) - return - } - - ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.FixedIP)) - } -} diff --git a/builder/openstack-new/step_create_image.go b/builder/openstack-new/step_create_image.go deleted file mode 100644 index b777e8b0b..000000000 --- a/builder/openstack-new/step_create_image.go +++ /dev/null @@ -1,84 +0,0 @@ -package openstack - -import ( - "fmt" - "log" - "time" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud" - "github.com/rackspace/gophercloud/openstack/compute/v2/images" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -type stepCreateImage struct{} - -func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { - config := state.Get("config").(Config) - server := state.Get("server").(*servers.Server) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - client, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - // Create the image - ui.Say(fmt.Sprintf("Creating the image: %s", config.ImageName)) - imageId, err := servers.CreateImage(client, server.ID, servers.CreateImageOpts{ - Name: config.ImageName, - }).ExtractImageID() - if err != nil { - err := fmt.Errorf("Error creating image: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - // Set the Image ID in the state - ui.Message(fmt.Sprintf("Image: %s", imageId)) - state.Put("image", imageId) - - // Wait for the image to become ready - ui.Say("Waiting for image to become ready...") - if err := WaitForImage(client, imageId); err != nil { - err := fmt.Errorf("Error waiting for image: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - return multistep.ActionContinue -} - -func (s *stepCreateImage) Cleanup(multistep.StateBag) { - // No cleanup... -} - -// WaitForImage waits for the given Image ID to become ready. -func WaitForImage(client *gophercloud.ServiceClient, imageId string) error { - for { - image, err := images.Get(client, imageId).Extract() - if err != nil { - errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) - if ok && errCode.Actual == 500 { - log.Printf("[ERROR] 500 error received, will ignore and retry: %s", err) - time.Sleep(2 * time.Second) - continue - } - - return err - } - - if image.Status == "ACTIVE" { - return nil - } - - log.Printf("Waiting for image creation status: %s (%d%%)", image.Status, image.Progress) - time.Sleep(2 * time.Second) - } -} diff --git a/builder/openstack-new/step_key_pair.go b/builder/openstack-new/step_key_pair.go deleted file mode 100644 index 06bcbf9ea..000000000 --- a/builder/openstack-new/step_key_pair.go +++ /dev/null @@ -1,106 +0,0 @@ -package openstack - -import ( - "fmt" - "os" - "runtime" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/common/uuid" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" -) - -type StepKeyPair struct { - Debug bool - DebugKeyPath string - keyName string -} - -func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { - config := state.Get("config").(Config) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - ui.Say("Creating temporary keypair for this instance...") - keyName := fmt.Sprintf("packer %s", uuid.TimeOrderedUUID()) - keypair, err := keypairs.Create(computeClient, keypairs.CreateOpts{ - Name: keyName, - }).Extract() - if err != nil { - state.Put("error", fmt.Errorf("Error creating temporary keypair: %s", err)) - return multistep.ActionHalt - } - - if keypair.PrivateKey == "" { - state.Put("error", fmt.Errorf("The temporary keypair returned was blank")) - return multistep.ActionHalt - } - - // If we're in debug mode, output the private key to the working - // directory. - if s.Debug { - ui.Message(fmt.Sprintf("Saving key for debug purposes: %s", s.DebugKeyPath)) - f, err := os.Create(s.DebugKeyPath) - if err != nil { - state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) - return multistep.ActionHalt - } - defer f.Close() - - // Write the key out - if _, err := f.Write([]byte(keypair.PrivateKey)); err != nil { - state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) - return multistep.ActionHalt - } - - // Chmod it so that it is SSH ready - if runtime.GOOS != "windows" { - if err := f.Chmod(0600); err != nil { - state.Put("error", fmt.Errorf("Error setting permissions of debug key: %s", err)) - return multistep.ActionHalt - } - } - } - - // Set the keyname so we know to delete it later - s.keyName = keyName - - // Set some state data for use in future steps - state.Put("keyPair", keyName) - state.Put("privateKey", keypair.PrivateKey) - - return multistep.ActionContinue -} - -func (s *StepKeyPair) Cleanup(state multistep.StateBag) { - // If no key name is set, then we never created it, so just return - if s.keyName == "" { - return - } - - config := state.Get("config").(Config) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - ui.Error(fmt.Sprintf( - "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) - return - } - - ui.Say("Deleting temporary keypair...") - err = keypairs.Delete(computeClient, s.keyName).ExtractErr() - if err != nil { - ui.Error(fmt.Sprintf( - "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) - } -} diff --git a/builder/openstack-new/step_run_source_server.go b/builder/openstack-new/step_run_source_server.go deleted file mode 100644 index 4432d5860..000000000 --- a/builder/openstack-new/step_run_source_server.go +++ /dev/null @@ -1,110 +0,0 @@ -package openstack - -import ( - "fmt" - "log" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -type StepRunSourceServer struct { - Flavor string - Name string - SourceImage string - SecurityGroups []string - Networks []string - - server *servers.Server -} - -func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction { - config := state.Get("config").(Config) - keyName := state.Get("keyPair").(string) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - networks := make([]servers.Network, len(s.Networks)) - for i, networkUuid := range s.Networks { - networks[i].UUID = networkUuid - } - - s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ - CreateOptsBuilder: servers.CreateOpts{ - Name: s.Name, - ImageRef: s.SourceImage, - FlavorName: s.Flavor, - SecurityGroups: s.SecurityGroups, - Networks: networks, - }, - - KeyName: keyName, - }).Extract() - if err != nil { - err := fmt.Errorf("Error launching source server: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - log.Printf("server id: %s", s.server.ID) - - ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.ID)) - stateChange := StateChangeConf{ - Pending: []string{"BUILD"}, - Target: "ACTIVE", - Refresh: ServerStateRefreshFunc(computeClient, s.server), - StepState: state, - } - latestServer, err := WaitForState(&stateChange) - if err != nil { - err := fmt.Errorf("Error waiting for server (%s) to become ready: %s", s.server.ID, err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - - s.server = latestServer.(*servers.Server) - state.Put("server", s.server) - - return multistep.ActionContinue -} - -func (s *StepRunSourceServer) Cleanup(state multistep.StateBag) { - if s.server == nil { - return - } - - config := state.Get("config").(Config) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) - return - } - - ui.Say("Terminating the source server...") - if err := servers.Delete(computeClient, s.server.ID).ExtractErr(); err != nil { - ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) - return - } - - stateChange := StateChangeConf{ - Pending: []string{"ACTIVE", "BUILD", "REBUILD", "SUSPENDED"}, - Refresh: ServerStateRefreshFunc(computeClient, s.server), - Target: "DELETED", - } - - WaitForState(&stateChange) -} diff --git a/builder/openstack-new/step_wait_for_rackconnect.go b/builder/openstack-new/step_wait_for_rackconnect.go deleted file mode 100644 index 6263bd17d..000000000 --- a/builder/openstack-new/step_wait_for_rackconnect.go +++ /dev/null @@ -1,52 +0,0 @@ -package openstack - -import ( - "fmt" - "time" - - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" - "github.com/rackspace/gophercloud/openstack/compute/v2/servers" -) - -type StepWaitForRackConnect struct { - Wait bool -} - -func (s *StepWaitForRackConnect) Run(state multistep.StateBag) multistep.StepAction { - if !s.Wait { - return multistep.ActionContinue - } - - config := state.Get("config").(Config) - server := state.Get("server").(*servers.Server) - ui := state.Get("ui").(packer.Ui) - - // We need the v2 compute client - computeClient, err := config.computeV2Client() - if err != nil { - err = fmt.Errorf("Error initializing compute client: %s", err) - state.Put("error", err) - return multistep.ActionHalt - } - - ui.Say(fmt.Sprintf( - "Waiting for server (%s) to become RackConnect ready...", server.ID)) - for { - server, err = servers.Get(computeClient, server.ID).Extract() - if err != nil { - return multistep.ActionHalt - } - - if server.Metadata["rackconnect_automation_status"] == "DEPLOYED" { - break - } - - time.Sleep(2 * time.Second) - } - - return multistep.ActionContinue -} - -func (s *StepWaitForRackConnect) Cleanup(state multistep.StateBag) { -} diff --git a/builder/openstack/access_config.go b/builder/openstack/access_config.go index cb1c9d7bd..e0f962c50 100644 --- a/builder/openstack/access_config.go +++ b/builder/openstack/access_config.go @@ -4,99 +4,106 @@ import ( "crypto/tls" "fmt" "net/http" - "net/url" "os" - "strings" - "github.com/mitchellh/gophercloud-fork-40444fb" - "github.com/mitchellh/packer/common" "github.com/mitchellh/packer/template/interpolate" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack" ) // AccessConfig is for common configuration related to openstack access type AccessConfig struct { - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - ApiKey string `mapstructure:"api_key"` - Project string `mapstructure:"project"` - Provider string `mapstructure:"provider"` - RawRegion string `mapstructure:"region"` - ProxyUrl string `mapstructure:"proxy_url"` - TenantId string `mapstructure:"tenant_id"` - Insecure bool `mapstructure:"insecure"` -} + Username string `mapstructure:"username"` + UserID string `mapstructure:"user_id"` + Password string `mapstructure:"password"` + APIKey string `mapstructure:"api_key"` + IdentityEndpoint string `mapstructure:"identity_endpoint"` + TenantID string `mapstructure:"tenant_id"` + TenantName string `mapstructure:"tenant_name"` + DomainID string `mapstructure:"domain_id"` + DomainName string `mapstructure:"domain_name"` + Insecure bool `mapstructure:"insecure"` + Region string `mapstructure:"region"` + EndpointType string `mapstructure:"endpoint_type"` -// Auth returns a valid Auth object for access to openstack services, or -// an error if the authentication couldn't be resolved. -func (c *AccessConfig) Auth() (gophercloud.AccessProvider, error) { - c.Username = common.ChooseString(c.Username, os.Getenv("SDK_USERNAME"), os.Getenv("OS_USERNAME")) - c.Password = common.ChooseString(c.Password, os.Getenv("SDK_PASSWORD"), os.Getenv("OS_PASSWORD")) - c.ApiKey = common.ChooseString(c.ApiKey, os.Getenv("SDK_API_KEY")) - c.Project = common.ChooseString(c.Project, os.Getenv("SDK_PROJECT"), os.Getenv("OS_TENANT_NAME")) - c.Provider = common.ChooseString(c.Provider, os.Getenv("SDK_PROVIDER"), os.Getenv("OS_AUTH_URL")) - c.RawRegion = common.ChooseString(c.RawRegion, os.Getenv("SDK_REGION"), os.Getenv("OS_REGION_NAME")) - c.TenantId = common.ChooseString(c.TenantId, os.Getenv("OS_TENANT_ID")) - - // OpenStack's auto-generated openrc.sh files do not append the suffix - // /tokens to the authentication URL. This ensures it is present when - // specifying the URL. - if strings.Contains(c.Provider, "://") && !strings.HasSuffix(c.Provider, "/tokens") { - c.Provider += "/tokens" - } - - authoptions := gophercloud.AuthOptions{ - AllowReauth: true, - - ApiKey: c.ApiKey, - TenantId: c.TenantId, - TenantName: c.Project, - Username: c.Username, - Password: c.Password, - } - - default_transport := &http.Transport{} - - if c.Insecure { - cfg := new(tls.Config) - cfg.InsecureSkipVerify = true - default_transport.TLSClientConfig = cfg - } - - // For corporate networks it may be the case where we want our API calls - // to be sent through a separate HTTP proxy than external traffic. - if c.ProxyUrl != "" { - url, err := url.Parse(c.ProxyUrl) - if err != nil { - return nil, err - } - - // The gophercloud.Context has a UseCustomClient method which - // would allow us to override with a new instance of http.Client. - default_transport.Proxy = http.ProxyURL(url) - } - - if c.Insecure || c.ProxyUrl != "" { - http.DefaultTransport = default_transport - } - - return gophercloud.Authenticate(c.Provider, authoptions) -} - -func (c *AccessConfig) Region() string { - return common.ChooseString(c.RawRegion, os.Getenv("SDK_REGION"), os.Getenv("OS_REGION_NAME")) + osClient *gophercloud.ProviderClient } func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { - errs := make([]error, 0) - if strings.HasPrefix(c.Provider, "rackspace") { - if c.Region() == "" { - errs = append(errs, fmt.Errorf("region must be specified when using rackspace")) + if c.EndpointType != "internal" && c.EndpointType != "internalURL" && + c.EndpointType != "admin" && c.EndpointType != "adminURL" && + c.EndpointType != "public" && c.EndpointType != "publicURL" && + c.EndpointType != "" { + return []error{fmt.Errorf("Invalid endpoint type provided")} + } + + if c.Region == "" { + c.Region = os.Getenv("OS_REGION_NAME") + } + + // Get as much as possible from the end + ao, err := openstack.AuthOptionsFromEnv() + if err != nil { + return []error{err} + } + + // Override values if we have them in our config + overrides := []struct { + From, To *string + }{ + {&c.Username, &ao.Username}, + {&c.UserID, &ao.UserID}, + {&c.Password, &ao.Password}, + {&c.APIKey, &ao.APIKey}, + {&c.IdentityEndpoint, &ao.IdentityEndpoint}, + {&c.TenantID, &ao.TenantID}, + {&c.TenantName, &ao.TenantName}, + {&c.DomainID, &ao.DomainID}, + {&c.DomainName, &ao.DomainName}, + } + for _, s := range overrides { + if *s.From != "" { + *s.To = *s.From } } - if len(errs) > 0 { - return errs + // Build the client itself + client, err := openstack.NewClient(ao.IdentityEndpoint) + if err != nil { + return []error{err} } + // If we have insecure set, then create a custom HTTP client that + // ignores SSL errors. + if c.Insecure { + config := &tls.Config{InsecureSkipVerify: true} + transport := &http.Transport{TLSClientConfig: config} + client.HTTPClient.Transport = transport + } + + // Auth + err = openstack.Authenticate(client, ao) + if err != nil { + return []error{err} + } + + c.osClient = client return nil } + +func (c *AccessConfig) computeV2Client() (*gophercloud.ServiceClient, error) { + return openstack.NewComputeV2(c.osClient, gophercloud.EndpointOpts{ + Region: c.Region, + Availability: c.getEndpointType(), + }) +} + +func (c *AccessConfig) getEndpointType() gophercloud.Availability { + if c.EndpointType == "internal" || c.EndpointType == "internalURL" { + return gophercloud.AvailabilityInternal + } + if c.EndpointType == "admin" || c.EndpointType == "adminURL" { + return gophercloud.AvailabilityAdmin + } + return gophercloud.AvailabilityPublic +} diff --git a/builder/openstack/access_config_test.go b/builder/openstack/access_config_test.go deleted file mode 100644 index cf37448cc..000000000 --- a/builder/openstack/access_config_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package openstack - -import ( - "os" - "testing" -) - -func init() { - // Clear out the openstack env vars so they don't - // affect our tests. - os.Setenv("SDK_REGION", "") - os.Setenv("OS_REGION_NAME", "") -} - -func testAccessConfig() *AccessConfig { - return &AccessConfig{} -} - -func TestAccessConfigPrepare_NoRegion_Rackspace(t *testing.T) { - c := testAccessConfig() - c.Provider = "rackspace-us" - if err := c.Prepare(nil); err == nil { - t.Fatalf("shouldn't have err: %s", err) - } -} - -func TestAccessConfigRegionWithEmptyEnv(t *testing.T) { - c := testAccessConfig() - c.Prepare(nil) - if c.Region() != "" { - t.Fatalf("Region should be empty") - } -} - -func TestAccessConfigRegionWithSdkRegionEnv(t *testing.T) { - c := testAccessConfig() - c.Prepare(nil) - - expectedRegion := "sdk_region" - os.Setenv("SDK_REGION", expectedRegion) - os.Setenv("OS_REGION_NAME", "") - if c.Region() != expectedRegion { - t.Fatalf("Region should be: %s", expectedRegion) - } -} - -func TestAccessConfigRegionWithOsRegionNameEnv(t *testing.T) { - c := testAccessConfig() - c.Prepare(nil) - - expectedRegion := "os_region_name" - os.Setenv("SDK_REGION", "") - os.Setenv("OS_REGION_NAME", expectedRegion) - if c.Region() != expectedRegion { - t.Fatalf("Region should be: %s", expectedRegion) - } -} - -func TestAccessConfigPrepare_NoRegion_PrivateCloud(t *testing.T) { - c := testAccessConfig() - c.Provider = "http://some-keystone-server:5000/v2.0" - if err := c.Prepare(nil); err != nil { - t.Fatalf("shouldn't have err: %s", err) - } -} - -func TestAccessConfigPrepare_Region(t *testing.T) { - dfw := "DFW" - c := testAccessConfig() - c.RawRegion = dfw - if err := c.Prepare(nil); err != nil { - t.Fatalf("shouldn't have err: %s", err) - } - if dfw != c.Region() { - t.Fatalf("Regions do not match: %s %s", dfw, c.Region()) - } -} diff --git a/builder/openstack/artifact.go b/builder/openstack/artifact.go index 6e75fad3e..aa60d2641 100644 --- a/builder/openstack/artifact.go +++ b/builder/openstack/artifact.go @@ -4,7 +4,8 @@ import ( "fmt" "log" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/images" ) // Artifact is an artifact implementation that contains built images. @@ -16,7 +17,7 @@ type Artifact struct { BuilderIdValue string // OpenStack connection for performing API stuff. - Conn gophercloud.CloudServersProvider + Client *gophercloud.ServiceClient } func (a *Artifact) BuilderId() string { @@ -42,5 +43,5 @@ func (a *Artifact) State(name string) interface{} { func (a *Artifact) Destroy() error { log.Printf("Destroying image: %s", a.ImageId) - return a.Conn.DeleteImageById(a.ImageId) + return images.Delete(a.Client, a.ImageId).ExtractErr() } diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index e6e5c6675..bebb28452 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -9,7 +9,6 @@ import ( "github.com/mitchellh/packer/common" "log" - "github.com/mitchellh/gophercloud-fork-40444fb" "github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/template/interpolate" @@ -55,28 +54,14 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { } func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { - auth, err := b.config.AccessConfig.Auth() + computeClient, err := b.config.computeV2Client() if err != nil { - return nil, err - } - //fetches the api requisites from gophercloud for the appropriate - //openstack variant - api, err := gophercloud.PopulateApi(b.config.RunConfig.OpenstackProvider) - if err != nil { - return nil, err - } - api.Region = b.config.AccessConfig.Region() - - csp, err := gophercloud.ServersApi(auth, api) - if err != nil { - log.Printf("Region: %s", b.config.AccessConfig.Region()) - return nil, err + return nil, fmt.Errorf("Error initializing compute client: %s", err) } // Setup the state bag and initial state for the steps state := new(multistep.BasicStateBag) state.Put("config", b.config) - state.Put("csp", csp) state.Put("hook", hook) state.Put("ui", ui) @@ -101,7 +86,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe FloatingIp: b.config.FloatingIp, }, &common.StepConnectSSH{ - SSHAddress: SSHAddress(csp, b.config.SSHInterface, b.config.SSHPort), + SSHAddress: SSHAddress(computeClient, b.config.SSHInterface, b.config.SSHPort), SSHConfig: SSHConfig(b.config.SSHUsername), SSHWaitTimeout: b.config.SSHTimeout(), }, @@ -135,7 +120,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe artifact := &Artifact{ ImageId: state.Get("image").(string), BuilderIdValue: BuilderId, - Conn: csp, + Client: computeClient, } return artifact, nil diff --git a/builder/openstack/server.go b/builder/openstack/server.go index ba22dd3e2..de8c9d103 100644 --- a/builder/openstack/server.go +++ b/builder/openstack/server.go @@ -3,12 +3,12 @@ package openstack import ( "errors" "fmt" - "github.com/mitchellh/multistep" - "github.com/racker/perigee" "log" "time" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) // StateRefreshFunc is a function type used for StateChangeConf that is @@ -33,21 +33,22 @@ type StateChangeConf struct { // ServerStateRefreshFunc returns a StateRefreshFunc that is used to watch // an openstack server. -func ServerStateRefreshFunc(csp gophercloud.CloudServersProvider, s *gophercloud.Server) StateRefreshFunc { +func ServerStateRefreshFunc( + client *gophercloud.ServiceClient, s *servers.Server) StateRefreshFunc { return func() (interface{}, string, int, error) { - resp, err := csp.ServerById(s.Id) + serverNew, err := servers.Get(client, s.ID).Extract() if err != nil { - urce, ok := err.(*perigee.UnexpectedResponseCodeError) - if ok && (urce.Actual == 404) { - log.Printf("404 on ServerStateRefresh, returning DELETED") - + errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) + if ok && errCode.Actual == 404 { + log.Printf("[INFO] 404 on ServerStateRefresh, returning DELETED") return nil, "DELETED", 0, nil } else { - log.Printf("Error on ServerStateRefresh: %s", err) + log.Printf("[ERROR] Error on ServerStateRefresh: %s", err) return nil, "", 0, err } } - return resp, resp.Status, resp.Progress, nil + + return serverNew, serverNew.Status, serverNew.Progress, nil } } diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go index d20f24170..7b0510f98 100644 --- a/builder/openstack/ssh.go +++ b/builder/openstack/ssh.go @@ -3,49 +3,67 @@ package openstack import ( "errors" "fmt" - "github.com/mitchellh/multistep" - "golang.org/x/crypto/ssh" + "log" "time" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" + "golang.org/x/crypto/ssh" ) // SSHAddress returns a function that can be given to the SSH communicator // for determining the SSH address based on the server AccessIPv4 setting.. -func SSHAddress(csp gophercloud.CloudServersProvider, sshinterface string, port int) func(multistep.StateBag) (string, error) { +func SSHAddress( + client *gophercloud.ServiceClient, + sshinterface string, port int) func(multistep.StateBag) (string, error) { return func(state multistep.StateBag) (string, error) { - s := state.Get("server").(*gophercloud.Server) + s := state.Get("server").(*servers.Server) - if ip := state.Get("access_ip").(gophercloud.FloatingIp); ip.Ip != "" { - return fmt.Sprintf("%s:%d", ip.Ip, port), nil + // If we have a floating IP, use that + ip := state.Get("access_ip").(*floatingip.FloatingIP) + if ip != nil && ip.FixedIP != "" { + return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil } - ip_pools, err := s.AllAddressPools() - if err != nil { - return "", errors.New("Error parsing SSH addresses") + if s.AccessIPv4 != "" { + return fmt.Sprintf("%s:%d", s.AccessIPv4, port), nil } - for pool, addresses := range ip_pools { - if sshinterface != "" { - if pool != sshinterface { - continue - } + + // Get all the addresses associated with this server. This + // was taken directly from Terraform. + for _, networkAddresses := range s.Addresses { + elements, ok := networkAddresses.([]interface{}) + if !ok { + log.Printf( + "[ERROR] Unknown return type for address field: %#v", + networkAddresses) + continue } - if pool != "" { - for _, address := range addresses { - if address.Addr != "" && address.Version == 4 { - return fmt.Sprintf("%s:%d", address.Addr, port), nil + + for _, element := range elements { + var addr string + address := element.(map[string]interface{}) + if address["OS-EXT-IPS:type"] == "floating" { + addr = address["addr"].(string) + } else { + if address["version"].(float64) == 4 { + addr = address["addr"].(string) } } + if addr != "" { + return fmt.Sprintf("%s:%d", addr, port), nil + } } } - serverState, err := csp.ServerById(s.Id) - + s, err := servers.Get(client, s.ID).Extract() if err != nil { return "", err } - state.Put("server", serverState) + state.Put("server", s) time.Sleep(1 * time.Second) return "", errors.New("couldn't determine IP address for server") diff --git a/builder/openstack/step_allocate_ip.go b/builder/openstack/step_allocate_ip.go index b64f8b617..16efe8d38 100644 --- a/builder/openstack/step_allocate_ip.go +++ b/builder/openstack/step_allocate_ip.go @@ -2,10 +2,11 @@ package openstack import ( "fmt" + "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" - - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/floatingip" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) type StepAllocateIp struct { @@ -15,53 +16,78 @@ type StepAllocateIp struct { func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { ui := state.Get("ui").(packer.Ui) - csp := state.Get("csp").(gophercloud.CloudServersProvider) - server := state.Get("server").(*gophercloud.Server) + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) - var instanceIp gophercloud.FloatingIp + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + var instanceIp *floatingip.FloatingIP // This is here in case we error out before putting instanceIp into the // statebag below, because it is requested by Cleanup() state.Put("access_ip", instanceIp) if s.FloatingIp != "" { - instanceIp.Ip = s.FloatingIp + *instanceIp = floatingip.FloatingIP{FixedIP: s.FloatingIp} } else if s.FloatingIpPool != "" { - newIp, err := csp.CreateFloatingIp(s.FloatingIpPool) + newIp, err := floatingip.Create(client, floatingip.CreateOpts{ + Pool: s.FloatingIpPool, + }).Extract() if err != nil { err := fmt.Errorf("Error creating floating ip from pool '%s'", s.FloatingIpPool) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - instanceIp = newIp - ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.Ip)) + + *instanceIp = *newIp + ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) } - if instanceIp.Ip != "" { - if err := csp.AssociateFloatingIp(server.Id, instanceIp); err != nil { - err := fmt.Errorf("Error associating floating IP %s with instance.", instanceIp.Ip) + if instanceIp != nil && instanceIp.FixedIP != "" { + err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() + if err != nil { + err := fmt.Errorf( + "Error associating floating IP %s with instance.", + instanceIp.FixedIP) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt - } else { - ui.Say(fmt.Sprintf("Added floating IP %s to instance...", instanceIp.Ip)) } + + ui.Say(fmt.Sprintf( + "Added floating IP %s to instance...", instanceIp.FixedIP)) } state.Put("access_ip", instanceIp) - return multistep.ActionContinue } func (s *StepAllocateIp) Cleanup(state multistep.StateBag) { + config := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) - csp := state.Get("csp").(gophercloud.CloudServersProvider) - instanceIp := state.Get("access_ip").(gophercloud.FloatingIp) - if s.FloatingIpPool != "" && instanceIp.Id != 0 { - if err := csp.DeleteFloatingIp(instanceIp); err != nil { - ui.Error(fmt.Sprintf("Error deleting temporary floating IP %s", instanceIp.Ip)) + instanceIp := state.Get("access_ip").(*floatingip.FloatingIP) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf( + "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + return + } + + if s.FloatingIpPool != "" && instanceIp.ID != "" { + if err := floatingip.Delete(client, instanceIp.ID).ExtractErr(); err != nil { + ui.Error(fmt.Sprintf( + "Error deleting temporary floating IP %s", instanceIp.FixedIP)) return } - ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.Ip)) + + ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.FixedIP)) } } diff --git a/builder/openstack/step_create_image.go b/builder/openstack/step_create_image.go index 52a2ec4d1..b777e8b0b 100644 --- a/builder/openstack/step_create_image.go +++ b/builder/openstack/step_create_image.go @@ -2,28 +2,36 @@ package openstack import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" "log" "time" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud" + "github.com/rackspace/gophercloud/openstack/compute/v2/images" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) type stepCreateImage struct{} func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { - csp := state.Get("csp").(gophercloud.CloudServersProvider) config := state.Get("config").(Config) - server := state.Get("server").(*gophercloud.Server) + server := state.Get("server").(*servers.Server) ui := state.Get("ui").(packer.Ui) + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + // Create the image ui.Say(fmt.Sprintf("Creating the image: %s", config.ImageName)) - createOpts := gophercloud.CreateImage{ + imageId, err := servers.CreateImage(client, server.ID, servers.CreateImageOpts{ Name: config.ImageName, - } - imageId, err := csp.CreateImage(server.Id, createOpts) + }).ExtractImageID() if err != nil { err := fmt.Errorf("Error creating image: %s", err) state.Put("error", err) @@ -32,12 +40,12 @@ func (s *stepCreateImage) Run(state multistep.StateBag) multistep.StepAction { } // Set the Image ID in the state - ui.Say(fmt.Sprintf("Image: %s", imageId)) + ui.Message(fmt.Sprintf("Image: %s", imageId)) state.Put("image", imageId) // Wait for the image to become ready ui.Say("Waiting for image to become ready...") - if err := WaitForImage(csp, imageId); err != nil { + if err := WaitForImage(client, imageId); err != nil { err := fmt.Errorf("Error waiting for image: %s", err) state.Put("error", err) ui.Error(err.Error()) @@ -52,10 +60,17 @@ func (s *stepCreateImage) Cleanup(multistep.StateBag) { } // WaitForImage waits for the given Image ID to become ready. -func WaitForImage(csp gophercloud.CloudServersProvider, imageId string) error { +func WaitForImage(client *gophercloud.ServiceClient, imageId string) error { for { - image, err := csp.ImageById(imageId) + image, err := images.Get(client, imageId).Extract() if err != nil { + errCode, ok := err.(*gophercloud.UnexpectedResponseCodeError) + if ok && errCode.Actual == 500 { + log.Printf("[ERROR] 500 error received, will ignore and retry: %s", err) + time.Sleep(2 * time.Second) + continue + } + return err } diff --git a/builder/openstack/step_key_pair.go b/builder/openstack/step_key_pair.go index 9c46b4377..06bcbf9ea 100644 --- a/builder/openstack/step_key_pair.go +++ b/builder/openstack/step_key_pair.go @@ -2,14 +2,13 @@ package openstack import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/common/uuid" - "github.com/mitchellh/packer/packer" - "log" "os" "runtime" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/common/uuid" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" ) type StepKeyPair struct { @@ -19,18 +18,28 @@ type StepKeyPair struct { } func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { - csp := state.Get("csp").(gophercloud.CloudServersProvider) + config := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + ui.Say("Creating temporary keypair for this instance...") keyName := fmt.Sprintf("packer %s", uuid.TimeOrderedUUID()) - log.Printf("temporary keypair name: %s", keyName) - keyResp, err := csp.CreateKeyPair(gophercloud.NewKeyPair{Name: keyName}) + keypair, err := keypairs.Create(computeClient, keypairs.CreateOpts{ + Name: keyName, + }).Extract() if err != nil { state.Put("error", fmt.Errorf("Error creating temporary keypair: %s", err)) return multistep.ActionHalt } - if keyResp.PrivateKey == "" { + + if keypair.PrivateKey == "" { state.Put("error", fmt.Errorf("The temporary keypair returned was blank")) return multistep.ActionHalt } @@ -47,7 +56,7 @@ func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { defer f.Close() // Write the key out - if _, err := f.Write([]byte(keyResp.PrivateKey)); err != nil { + if _, err := f.Write([]byte(keypair.PrivateKey)); err != nil { state.Put("error", fmt.Errorf("Error saving debug key: %s", err)) return multistep.ActionHalt } @@ -66,7 +75,7 @@ func (s *StepKeyPair) Run(state multistep.StateBag) multistep.StepAction { // Set some state data for use in future steps state.Put("keyPair", keyName) - state.Put("privateKey", keyResp.PrivateKey) + state.Put("privateKey", keypair.PrivateKey) return multistep.ActionContinue } @@ -77,11 +86,19 @@ func (s *StepKeyPair) Cleanup(state multistep.StateBag) { return } - csp := state.Get("csp").(gophercloud.CloudServersProvider) + config := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf( + "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) + return + } + ui.Say("Deleting temporary keypair...") - err := csp.DeleteKeyPair(s.keyName) + err = keypairs.Delete(computeClient, s.keyName).ExtractErr() if err != nil { ui.Error(fmt.Sprintf( "Error cleaning up keypair. Please delete the key manually: %s", s.keyName)) diff --git a/builder/openstack/step_run_source_server.go b/builder/openstack/step_run_source_server.go index 19e7f024d..4432d5860 100644 --- a/builder/openstack/step_run_source_server.go +++ b/builder/openstack/step_run_source_server.go @@ -2,11 +2,12 @@ package openstack import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" "log" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/extensions/keypairs" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) type StepRunSourceServer struct { @@ -16,37 +17,38 @@ type StepRunSourceServer struct { SecurityGroups []string Networks []string - server *gophercloud.Server + server *servers.Server } func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction { - csp := state.Get("csp").(gophercloud.CloudServersProvider) + config := state.Get("config").(Config) keyName := state.Get("keyPair").(string) ui := state.Get("ui").(packer.Ui) - // XXX - validate image and flavor is available - - securityGroups := make([]map[string]interface{}, len(s.SecurityGroups)) - for i, groupName := range s.SecurityGroups { - securityGroups[i] = make(map[string]interface{}) - securityGroups[i]["name"] = groupName + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt } - networks := make([]gophercloud.NetworkConfig, len(s.Networks)) + networks := make([]servers.Network, len(s.Networks)) for i, networkUuid := range s.Networks { - networks[i].Uuid = networkUuid + networks[i].UUID = networkUuid } - server := gophercloud.NewServer{ - Name: s.Name, - ImageRef: s.SourceImage, - FlavorRef: s.Flavor, - KeyPairName: keyName, - SecurityGroup: securityGroups, - Networks: networks, - } + s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ + CreateOptsBuilder: servers.CreateOpts{ + Name: s.Name, + ImageRef: s.SourceImage, + FlavorName: s.Flavor, + SecurityGroups: s.SecurityGroups, + Networks: networks, + }, - serverResp, err := csp.CreateServer(server) + KeyName: keyName, + }).Extract() if err != nil { err := fmt.Errorf("Error launching source server: %s", err) state.Put("error", err) @@ -54,25 +56,24 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction return multistep.ActionHalt } - s.server, err = csp.ServerById(serverResp.Id) - log.Printf("server id: %s", s.server.Id) + log.Printf("server id: %s", s.server.ID) - ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.Id)) + ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.ID)) stateChange := StateChangeConf{ Pending: []string{"BUILD"}, Target: "ACTIVE", - Refresh: ServerStateRefreshFunc(csp, s.server), + Refresh: ServerStateRefreshFunc(computeClient, s.server), StepState: state, } latestServer, err := WaitForState(&stateChange) if err != nil { - err := fmt.Errorf("Error waiting for server (%s) to become ready: %s", s.server.Id, err) + err := fmt.Errorf("Error waiting for server (%s) to become ready: %s", s.server.ID, err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - s.server = latestServer.(*gophercloud.Server) + s.server = latestServer.(*servers.Server) state.Put("server", s.server) return multistep.ActionContinue @@ -83,18 +84,25 @@ func (s *StepRunSourceServer) Cleanup(state multistep.StateBag) { return } - csp := state.Get("csp").(gophercloud.CloudServersProvider) + config := state.Get("config").(Config) ui := state.Get("ui").(packer.Ui) + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) + return + } + ui.Say("Terminating the source server...") - if err := csp.DeleteServerById(s.server.Id); err != nil { + if err := servers.Delete(computeClient, s.server.ID).ExtractErr(); err != nil { ui.Error(fmt.Sprintf("Error terminating server, may still be around: %s", err)) return } stateChange := StateChangeConf{ Pending: []string{"ACTIVE", "BUILD", "REBUILD", "SUSPENDED"}, - Refresh: ServerStateRefreshFunc(csp, s.server), + Refresh: ServerStateRefreshFunc(computeClient, s.server), Target: "DELETED", } diff --git a/builder/openstack/step_wait_for_rackconnect.go b/builder/openstack/step_wait_for_rackconnect.go index ee6ee6138..6263bd17d 100644 --- a/builder/openstack/step_wait_for_rackconnect.go +++ b/builder/openstack/step_wait_for_rackconnect.go @@ -2,11 +2,11 @@ package openstack import ( "fmt" - "github.com/mitchellh/multistep" - "github.com/mitchellh/packer/packer" "time" - "github.com/mitchellh/gophercloud-fork-40444fb" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/servers" ) type StepWaitForRackConnect struct { @@ -18,14 +18,22 @@ func (s *StepWaitForRackConnect) Run(state multistep.StateBag) multistep.StepAct return multistep.ActionContinue } - csp := state.Get("csp").(gophercloud.CloudServersProvider) - server := state.Get("server").(*gophercloud.Server) + config := state.Get("config").(Config) + server := state.Get("server").(*servers.Server) ui := state.Get("ui").(packer.Ui) - ui.Say(fmt.Sprintf("Waiting for server (%s) to become RackConnect ready...", server.Id)) + // We need the v2 compute client + computeClient, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + ui.Say(fmt.Sprintf( + "Waiting for server (%s) to become RackConnect ready...", server.ID)) for { - server, err := csp.ServerById(server.Id) + server, err = servers.Get(computeClient, server.ID).Extract() if err != nil { return multistep.ActionHalt } From c80d1ab46b609637399a2e696f1e184812b92067 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:16:56 -0400 Subject: [PATCH 71/85] remove the new plugin --- plugin/builder-openstack-new/main.go | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 plugin/builder-openstack-new/main.go diff --git a/plugin/builder-openstack-new/main.go b/plugin/builder-openstack-new/main.go deleted file mode 100644 index d8075c78d..000000000 --- a/plugin/builder-openstack-new/main.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -import ( - "github.com/mitchellh/packer/builder/openstack-new" - "github.com/mitchellh/packer/packer/plugin" -) - -func main() { - server, err := plugin.Server() - if err != nil { - panic(err) - } - server.RegisterBuilder(new(openstack.Builder)) - server.Serve() -} From 50e2eb30e6f2c0cb2313b91c1ccb9db62c7eb2e9 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:28:38 -0400 Subject: [PATCH 72/85] builder/openstack: modifications to work with rackspace --- builder/openstack/access_config.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/builder/openstack/access_config.go b/builder/openstack/access_config.go index e0f962c50..71679b979 100644 --- a/builder/openstack/access_config.go +++ b/builder/openstack/access_config.go @@ -42,10 +42,7 @@ func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { } // Get as much as possible from the end - ao, err := openstack.AuthOptionsFromEnv() - if err != nil { - return []error{err} - } + ao, _ := openstack.AuthOptionsFromEnv() // Override values if we have them in our config overrides := []struct { From e724b5fe80de907da0dc8aa15b10ae0b83e0a812 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:33:52 -0400 Subject: [PATCH 73/85] builder/openstack: support legacy env vars --- builder/openstack/access_config.go | 17 ++++++ .../docs/builders/openstack.html.markdown | 57 +++++-------------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/builder/openstack/access_config.go b/builder/openstack/access_config.go index 71679b979..ca33495a8 100644 --- a/builder/openstack/access_config.go +++ b/builder/openstack/access_config.go @@ -41,6 +41,23 @@ func (c *AccessConfig) Prepare(ctx *interpolate.Context) []error { c.Region = os.Getenv("OS_REGION_NAME") } + // Legacy RackSpace stuff. We're keeping this around to keep things BC. + if c.APIKey == "" { + c.APIKey = os.Getenv("SDK_API_KEY") + } + if c.Password == "" { + c.Password = os.Getenv("SDK_PASSWORD") + } + if c.Region == "" { + c.Region = os.Getenv("SDK_REGION") + } + if c.TenantName == "" { + c.TenantName = os.Getenv("SDK_PROJECT") + } + if c.Username == "" { + c.Username = os.Getenv("SDK_USERNAME") + } + // Get as much as possible from the end ao, _ := openstack.AuthOptionsFromEnv() diff --git a/website/source/docs/builders/openstack.html.markdown b/website/source/docs/builders/openstack.html.markdown index d5dbbf249..10f38a445 100644 --- a/website/source/docs/builders/openstack.html.markdown +++ b/website/source/docs/builders/openstack.html.markdown @@ -34,23 +34,21 @@ each category, the available configuration keys are alphabetized. * `image_name` (string) - The name of the resulting image. -* `password` (string) - The password used to connect to the OpenStack service. - If not specified, Packer will use the environment variables - `SDK_PASSWORD` or `OS_PASSWORD` (in that order), if set. - * `source_image` (string) - The ID or full URL to the base image to use. This is the image that will be used to launch a new server and provision it. * `username` (string) - The username used to connect to the OpenStack service. + If not specified, Packer will use the environment variable + `OS_USERNAME`, if set. + +* `password` (string) - The password used to connect to the OpenStack service. If not specified, Packer will use the environment variables - `SDK_USERNAME` or `OS_USERNAME` (in that order), if set. + `OS_PASSWORD`, if set. ### Optional: * `api_key` (string) - The API key used to access OpenStack. Some OpenStack installations require this. - If not specified, Packer will use the environment variables - `SDK_API_KEY`, if set. * `floating_ip` (string) - A specific floating IP to assign to this instance. `use_floating_ip` must also be set to true for this to have an affect. @@ -65,32 +63,18 @@ each category, the available configuration keys are alphabetized. * `networks` (array of strings) - A list of networks by UUID to attach to this instance. -* `openstack_provider` (string) - A name of a provider that has a slightly - different API model. Currently supported values are "openstack" (default), - and "rackspace". - -* `project` (string) - The project name to boot the instance into. Some - OpenStack installations require this. - If not specified, Packer will use the environment variables - `SDK_PROJECT` or `OS_TENANT_NAME` (in that order), if set. - -* `provider` (string) - The provider used to connect to the OpenStack service. - If not specified, Packer will use the environment variables `SDK_PROVIDER` - or `OS_AUTH_URL` (in that order), if set. - For Rackspace this should be `rackspace-us` or `rackspace-uk`. - -* `proxy_url` (string) +* `tenant_id` or `tenant_name` (string) - The tenant ID or name to boot the + instance into. Some OpenStack installations require this. + If not specified, Packer will use the environment variable + `OS_TENANT_NAME`, if set. * `security_groups` (array of strings) - A list of security groups by name to add to this instance. * `region` (string) - The name of the region, such as "DFW", in which to launch the server to create the AMI. - If not specified, Packer will use the environment variables - `SDK_REGION` or `OS_REGION_NAME` (in that order), if set. - For a `provider` of "rackspace", it is required to specify a region, - either using this option or with an environment variable. For other - providers, including a private cloud, specifying a region is optional. + If not specified, Packer will use the environment variable + `OS_REGION_NAME`, if set. * `ssh_port` (integer) - The port that SSH will be available on. Defaults to port 22. @@ -106,9 +90,6 @@ each category, the available configuration keys are alphabetized. useful for Rackspace are "public" or "private", and the default behavior is to connect via whichever is returned first from the OpenStack API. -* `tenant_id` (string) - Tenant ID for accessing OpenStack if your - installation requires this. - * `use_floating_ip` (boolean) - Whether or not to use a floating IP for the instance. Defaults to false. @@ -124,10 +105,8 @@ Ubuntu 12.04 LTS (Precise Pangolin) on Rackspace OpenStack cloud offering. ```javascript { "type": "openstack", - "username": "", - "api_key": "", - "openstack_provider": "rackspace", - "provider": "rackspace-us", + "username": "foo", + "password": "foo", "region": "DFW", "ssh_username": "root", "image_name": "Test image", @@ -160,13 +139,3 @@ script is setting environment variables like: * `OS_TENANT_ID` * `OS_USERNAME` * `OS_PASSWORD` - -## Troubleshooting - -*I get the error "Missing or incorrect provider"* - -* Verify your "username", "password" and "provider" settings. - -*I get the error "Missing endpoint, or insufficient privileges to access endpoint"* - -* Verify your "region" setting. From 590177ea4b0b5057c5ec9a4da4b2811651d62c8b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 00:35:54 -0400 Subject: [PATCH 74/85] builder/openstack: fix unit tests --- builder/openstack/builder_test.go | 53 ------------------------------- 1 file changed, 53 deletions(-) diff --git a/builder/openstack/builder_test.go b/builder/openstack/builder_test.go index badf9784d..ce15873eb 100644 --- a/builder/openstack/builder_test.go +++ b/builder/openstack/builder_test.go @@ -9,7 +9,6 @@ func testConfig() map[string]interface{} { return map[string]interface{}{ "username": "foo", "password": "bar", - "provider": "foo", "region": "DFW", "image_name": "foo", "source_image": "foo", @@ -40,55 +39,3 @@ func TestBuilder_Prepare_BadType(t *testing.T) { t.Fatalf("prepare should fail") } } - -func TestBuilderPrepare_ImageName(t *testing.T) { - var b Builder - config := testConfig() - - // Test good - config["image_name"] = "foo" - warns, err := b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err != nil { - t.Fatalf("should not have error: %s", err) - } - - // Test bad - config["image_name"] = "foo {{" - b = Builder{} - warns, err = b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } - - // Test bad - delete(config, "image_name") - b = Builder{} - warns, err = b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } -} - -func TestBuilderPrepare_InvalidKey(t *testing.T) { - var b Builder - config := testConfig() - - // Add a random key - config["i_should_not_be_valid"] = true - warns, err := b.Prepare(config) - if len(warns) > 0 { - t.Fatalf("bad: %#v", warns) - } - if err == nil { - t.Fatal("should have error") - } -} From 5d32a1f6e059b107a112ec7811dd8a16c2e6101f Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:02:04 -0400 Subject: [PATCH 75/85] builder/openstack: use IP not FixedIP --- builder/openstack/ssh.go | 4 ++-- builder/openstack/step_allocate_ip.go | 27 ++++++++++++++------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/builder/openstack/ssh.go b/builder/openstack/ssh.go index 7b0510f98..76c2686b1 100644 --- a/builder/openstack/ssh.go +++ b/builder/openstack/ssh.go @@ -23,8 +23,8 @@ func SSHAddress( // If we have a floating IP, use that ip := state.Get("access_ip").(*floatingip.FloatingIP) - if ip != nil && ip.FixedIP != "" { - return fmt.Sprintf("%s:%d", ip.FixedIP, port), nil + if ip != nil && ip.IP != "" { + return fmt.Sprintf("%s:%d", ip.IP, port), nil } if s.AccessIPv4 != "" { diff --git a/builder/openstack/step_allocate_ip.go b/builder/openstack/step_allocate_ip.go index 16efe8d38..0ab9b3529 100644 --- a/builder/openstack/step_allocate_ip.go +++ b/builder/openstack/step_allocate_ip.go @@ -27,13 +27,14 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - var instanceIp *floatingip.FloatingIP + var instanceIp floatingip.FloatingIP + // This is here in case we error out before putting instanceIp into the // statebag below, because it is requested by Cleanup() - state.Put("access_ip", instanceIp) + state.Put("access_ip", &instanceIp) if s.FloatingIp != "" { - *instanceIp = floatingip.FloatingIP{FixedIP: s.FloatingIp} + instanceIp.IP = s.FloatingIp } else if s.FloatingIpPool != "" { newIp, err := floatingip.Create(client, floatingip.CreateOpts{ Pool: s.FloatingIpPool, @@ -45,26 +46,26 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - *instanceIp = *newIp - ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.FixedIP)) + instanceIp = *newIp + ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.IP)) } - if instanceIp != nil && instanceIp.FixedIP != "" { - err := floatingip.Associate(client, server.ID, instanceIp.FixedIP).ExtractErr() + if instanceIp.IP != "" { + err := floatingip.Associate(client, server.ID, instanceIp.IP).ExtractErr() if err != nil { err := fmt.Errorf( "Error associating floating IP %s with instance.", - instanceIp.FixedIP) + instanceIp.IP) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } ui.Say(fmt.Sprintf( - "Added floating IP %s to instance...", instanceIp.FixedIP)) + "Added floating IP %s to instance...", instanceIp.IP)) } - state.Put("access_ip", instanceIp) + state.Put("access_ip", &instanceIp) return multistep.ActionContinue } @@ -77,17 +78,17 @@ func (s *StepAllocateIp) Cleanup(state multistep.StateBag) { client, err := config.computeV2Client() if err != nil { ui.Error(fmt.Sprintf( - "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + "Error deleting temporary floating IP %s", instanceIp.IP)) return } if s.FloatingIpPool != "" && instanceIp.ID != "" { if err := floatingip.Delete(client, instanceIp.ID).ExtractErr(); err != nil { ui.Error(fmt.Sprintf( - "Error deleting temporary floating IP %s", instanceIp.FixedIP)) + "Error deleting temporary floating IP %s", instanceIp.IP)) return } - ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.FixedIP)) + ui.Say(fmt.Sprintf("Deleted temporary floating IP %s", instanceIp.IP)) } } From ad374e82afe184249687a8e0993e7376b9d92a7c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:05:03 -0400 Subject: [PATCH 76/85] builder/openstack: shuffle some fields to note unused fields --- builder/openstack/run_config.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/builder/openstack/run_config.go b/builder/openstack/run_config.go index e5d73c9c1..de76fad20 100644 --- a/builder/openstack/run_config.go +++ b/builder/openstack/run_config.go @@ -11,19 +11,21 @@ import ( // RunConfig contains configuration for running an instance from a source // image and details on how to access that launched image. type RunConfig struct { - SourceImage string `mapstructure:"source_image"` - Flavor string `mapstructure:"flavor"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort int `mapstructure:"ssh_port"` - SSHInterface string `mapstructure:"ssh_interface"` - OpenstackProvider string `mapstructure:"openstack_provider"` - UseFloatingIp bool `mapstructure:"use_floating_ip"` - RackconnectWait bool `mapstructure:"rackconnect_wait"` - FloatingIpPool string `mapstructure:"floating_ip_pool"` - FloatingIp string `mapstructure:"floating_ip"` - SecurityGroups []string `mapstructure:"security_groups"` - Networks []string `mapstructure:"networks"` + SourceImage string `mapstructure:"source_image"` + Flavor string `mapstructure:"flavor"` + RawSSHTimeout string `mapstructure:"ssh_timeout"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort int `mapstructure:"ssh_port"` + SSHInterface string `mapstructure:"ssh_interface"` + RackconnectWait bool `mapstructure:"rackconnect_wait"` + FloatingIpPool string `mapstructure:"floating_ip_pool"` + FloatingIp string `mapstructure:"floating_ip"` + SecurityGroups []string `mapstructure:"security_groups"` + Networks []string `mapstructure:"networks"` + + // Not really used, but here for BC + OpenstackProvider string `mapstructure:"openstack_provider"` + UseFloatingIp bool `mapstructure:"use_floating_ip"` // Unexported fields that are calculated from others sshTimeout time.Duration From 92b6b5c387b4f090d55450b44ba1619d5a3eddef Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:32:31 -0400 Subject: [PATCH 77/85] builder/openstack: can ref flavor by name --- builder/openstack/builder.go | 4 +- builder/openstack/step_load_flavor.go | 61 +++++++++++++++++++ builder/openstack/step_run_source_server.go | 8 ++- .../docs/builders/openstack.html.markdown | 2 +- 4 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 builder/openstack/step_load_flavor.go diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index bebb28452..d30ede8d3 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -67,13 +67,15 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe // Build the steps steps := []multistep.Step{ + &StepLoadFlavor{ + Flavor: b.config.Flavor, + }, &StepKeyPair{ Debug: b.config.PackerDebug, DebugKeyPath: fmt.Sprintf("os_%s.pem", b.config.PackerBuildName), }, &StepRunSourceServer{ Name: b.config.ImageName, - Flavor: b.config.Flavor, SourceImage: b.config.SourceImage, SecurityGroups: b.config.SecurityGroups, Networks: b.config.Networks, diff --git a/builder/openstack/step_load_flavor.go b/builder/openstack/step_load_flavor.go new file mode 100644 index 000000000..8b8cae994 --- /dev/null +++ b/builder/openstack/step_load_flavor.go @@ -0,0 +1,61 @@ +package openstack + +import ( + "fmt" + "log" + + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "github.com/rackspace/gophercloud/openstack/compute/v2/flavors" +) + +// StepLoadFlavor gets the FlavorRef from a Flavor. It first assumes +// that the Flavor is a ref and verifies it. Otherwise, it tries to find +// the flavor by name. +type StepLoadFlavor struct { + Flavor string +} + +func (s *StepLoadFlavor) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(Config) + ui := state.Get("ui").(packer.Ui) + + // We need the v2 compute client + client, err := config.computeV2Client() + if err != nil { + err = fmt.Errorf("Error initializing compute client: %s", err) + state.Put("error", err) + return multistep.ActionHalt + } + + ui.Say(fmt.Sprintf("Loading flavor: %s", s.Flavor)) + log.Printf("[INFO] Loading flavor by ID: %s", s.Flavor) + flavor, err := flavors.Get(client, s.Flavor).Extract() + if err != nil { + log.Printf("[ERROR] Failed to find flavor by ID: %s", err) + geterr := err + + log.Printf("[INFO] Loading flavor by name: %s", s.Flavor) + id, err := flavors.IDFromName(client, s.Flavor) + if err != nil { + log.Printf("[ERROR] Failed to find flavor by name: %s", err) + err = fmt.Errorf( + "Unable to find specified flavor by ID or name!\n\n"+ + "Error from ID lookup: %s\n\n"+ + "Error from name lookup: %s", + geterr, + err) + state.Put("error", err) + return multistep.ActionHalt + } + + flavor = &flavors.Flavor{ID: id} + } + + ui.Message(fmt.Sprintf("Verified flavor. ID: %s", flavor.ID)) + state.Put("flavor_id", flavor.ID) + return multistep.ActionContinue +} + +func (s *StepLoadFlavor) Cleanup(state multistep.StateBag) { +} diff --git a/builder/openstack/step_run_source_server.go b/builder/openstack/step_run_source_server.go index 4432d5860..89d816297 100644 --- a/builder/openstack/step_run_source_server.go +++ b/builder/openstack/step_run_source_server.go @@ -11,7 +11,6 @@ import ( ) type StepRunSourceServer struct { - Flavor string Name string SourceImage string SecurityGroups []string @@ -22,6 +21,7 @@ type StepRunSourceServer struct { func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction { config := state.Get("config").(Config) + flavor := state.Get("flavor_id").(string) keyName := state.Get("keyPair").(string) ui := state.Get("ui").(packer.Ui) @@ -38,11 +38,12 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction networks[i].UUID = networkUuid } + ui.Say("Launching server...") s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ CreateOptsBuilder: servers.CreateOpts{ Name: s.Name, ImageRef: s.SourceImage, - FlavorName: s.Flavor, + FlavorRef: flavor, SecurityGroups: s.SecurityGroups, Networks: networks, }, @@ -56,9 +57,10 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction return multistep.ActionHalt } + ui.Message(fmt.Sprintf("Server ID: %s", s.server.ID)) log.Printf("server id: %s", s.server.ID) - ui.Say(fmt.Sprintf("Waiting for server (%s) to become ready...", s.server.ID)) + ui.Say("Waiting for server to become ready...") stateChange := StateChangeConf{ Pending: []string{"BUILD"}, Target: "ACTIVE", diff --git a/website/source/docs/builders/openstack.html.markdown b/website/source/docs/builders/openstack.html.markdown index 10f38a445..3b5bf791d 100644 --- a/website/source/docs/builders/openstack.html.markdown +++ b/website/source/docs/builders/openstack.html.markdown @@ -29,7 +29,7 @@ each category, the available configuration keys are alphabetized. ### Required: -* `flavor` (string) - The ID or full URL for the desired flavor for the +* `flavor` (string) - The ID, name, or full URL for the desired flavor for the server to be created. * `image_name` (string) - The name of the resulting image. From 53b117e223ab27f3dec38ca34b22caea4587273e Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:54:12 -0400 Subject: [PATCH 78/85] update CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1bc247f5..15ce0290a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,12 @@ BACKWARDS INCOMPATIBILITIES: - * The DigitalOcean builder no longer supports the v1 API which has been + * builder/digitalocean: no longer supports the v1 API which has been deprecated for some time. Most configurations should continue to work as long as you use the `api_token` field for auth. * builder/digitalocean: `image`, `region`, and `size` are now required. + * builder/openstack: auth parameters have been changed to better + reflect OS terminology. Existing environment variables still work. FEATURES: From 44d86c2e90faa8876b3ac99e7beaf67021c0001a Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 10:54:36 -0400 Subject: [PATCH 79/85] update CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15ce0290a..d077ceba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,9 @@ IMPROVEMENTS: * builder/parallels: Support Parallels Desktop 11 [GH-2199] * builder/openstack: Add `rackconnect_wait` for Rackspace customers to wait for RackConnect data to appear - * buidler/openstakc: Add `ssh_interface` option for rackconnect for users that + * buidler/openstack: Add `ssh_interface` option for rackconnect for users that have prohibitive firewalls + * builder/openstack: Flavor names can be used as well as refs * builder/virtualbox: Added option: `ssh_skip_nat_mapping` to skip the automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support From 693f04afccd604e41b07b261ecea873990bc046c Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 11:10:10 -0400 Subject: [PATCH 80/85] builder/openstack: AZ support --- builder/openstack/builder.go | 9 ++++---- builder/openstack/run_config.go | 23 ++++++++++--------- builder/openstack/step_run_source_server.go | 20 ++++++++-------- .../docs/builders/openstack.html.markdown | 4 ++++ 4 files changed, 32 insertions(+), 24 deletions(-) diff --git a/builder/openstack/builder.go b/builder/openstack/builder.go index d30ede8d3..ab60afc0e 100644 --- a/builder/openstack/builder.go +++ b/builder/openstack/builder.go @@ -75,10 +75,11 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe DebugKeyPath: fmt.Sprintf("os_%s.pem", b.config.PackerBuildName), }, &StepRunSourceServer{ - Name: b.config.ImageName, - SourceImage: b.config.SourceImage, - SecurityGroups: b.config.SecurityGroups, - Networks: b.config.Networks, + Name: b.config.ImageName, + SourceImage: b.config.SourceImage, + SecurityGroups: b.config.SecurityGroups, + Networks: b.config.Networks, + AvailabilityZone: b.config.AvailabilityZone, }, &StepWaitForRackConnect{ Wait: b.config.RackconnectWait, diff --git a/builder/openstack/run_config.go b/builder/openstack/run_config.go index de76fad20..4a6a1b81f 100644 --- a/builder/openstack/run_config.go +++ b/builder/openstack/run_config.go @@ -11,17 +11,18 @@ import ( // RunConfig contains configuration for running an instance from a source // image and details on how to access that launched image. type RunConfig struct { - SourceImage string `mapstructure:"source_image"` - Flavor string `mapstructure:"flavor"` - RawSSHTimeout string `mapstructure:"ssh_timeout"` - SSHUsername string `mapstructure:"ssh_username"` - SSHPort int `mapstructure:"ssh_port"` - SSHInterface string `mapstructure:"ssh_interface"` - RackconnectWait bool `mapstructure:"rackconnect_wait"` - FloatingIpPool string `mapstructure:"floating_ip_pool"` - FloatingIp string `mapstructure:"floating_ip"` - SecurityGroups []string `mapstructure:"security_groups"` - Networks []string `mapstructure:"networks"` + SourceImage string `mapstructure:"source_image"` + Flavor string `mapstructure:"flavor"` + RawSSHTimeout string `mapstructure:"ssh_timeout"` + SSHUsername string `mapstructure:"ssh_username"` + SSHPort int `mapstructure:"ssh_port"` + SSHInterface string `mapstructure:"ssh_interface"` + AvailabilityZone string `mapstructure:"availability_zone"` + RackconnectWait bool `mapstructure:"rackconnect_wait"` + FloatingIpPool string `mapstructure:"floating_ip_pool"` + FloatingIp string `mapstructure:"floating_ip"` + SecurityGroups []string `mapstructure:"security_groups"` + Networks []string `mapstructure:"networks"` // Not really used, but here for BC OpenstackProvider string `mapstructure:"openstack_provider"` diff --git a/builder/openstack/step_run_source_server.go b/builder/openstack/step_run_source_server.go index 89d816297..4014f1d95 100644 --- a/builder/openstack/step_run_source_server.go +++ b/builder/openstack/step_run_source_server.go @@ -11,10 +11,11 @@ import ( ) type StepRunSourceServer struct { - Name string - SourceImage string - SecurityGroups []string - Networks []string + Name string + SourceImage string + SecurityGroups []string + Networks []string + AvailabilityZone string server *servers.Server } @@ -41,11 +42,12 @@ func (s *StepRunSourceServer) Run(state multistep.StateBag) multistep.StepAction ui.Say("Launching server...") s.server, err = servers.Create(computeClient, keypairs.CreateOptsExt{ CreateOptsBuilder: servers.CreateOpts{ - Name: s.Name, - ImageRef: s.SourceImage, - FlavorRef: flavor, - SecurityGroups: s.SecurityGroups, - Networks: networks, + Name: s.Name, + ImageRef: s.SourceImage, + FlavorRef: flavor, + SecurityGroups: s.SecurityGroups, + Networks: networks, + AvailabilityZone: s.AvailabilityZone, }, KeyName: keyName, diff --git a/website/source/docs/builders/openstack.html.markdown b/website/source/docs/builders/openstack.html.markdown index 3b5bf791d..a7fef29dc 100644 --- a/website/source/docs/builders/openstack.html.markdown +++ b/website/source/docs/builders/openstack.html.markdown @@ -50,6 +50,10 @@ each category, the available configuration keys are alphabetized. * `api_key` (string) - The API key used to access OpenStack. Some OpenStack installations require this. +* `availability_zone` (string) - The availability zone to launch the + server in. If this isn't specified, the default enforced by your OpenStack + cluster will be used. This may be required for some OpenStack clusters. + * `floating_ip` (string) - A specific floating IP to assign to this instance. `use_floating_ip` must also be set to true for this to have an affect. From 9bb722c99f3f2e3da83bfaf72d4411e754202644 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 11:10:34 -0400 Subject: [PATCH 81/85] update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d077ceba6..b96154c2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ IMPROVEMENTS: * buidler/openstack: Add `ssh_interface` option for rackconnect for users that have prohibitive firewalls * builder/openstack: Flavor names can be used as well as refs + * builder/openstack: Add `availability_zone` [GH-2016] * builder/virtualbox: Added option: `ssh_skip_nat_mapping` to skip the automatic port forward for SSH and to use the guest port directly. [GH-1078] * builder/virtualbox: Added SCSI support From 35b8df18162f8849eb28f1bed53f67c1421aacd4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 11:14:28 -0400 Subject: [PATCH 82/85] website: note cloud-init req for openstack [GH-1750] --- website/source/docs/builders/openstack.html.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/source/docs/builders/openstack.html.markdown b/website/source/docs/builders/openstack.html.markdown index a7fef29dc..fcd210dec 100644 --- a/website/source/docs/builders/openstack.html.markdown +++ b/website/source/docs/builders/openstack.html.markdown @@ -36,6 +36,8 @@ each category, the available configuration keys are alphabetized. * `source_image` (string) - The ID or full URL to the base image to use. This is the image that will be used to launch a new server and provision it. + Unless you specify completely custom SSH settings, the source image must + have `cloud-init` installed so that the keypair gets assigned properly. * `username` (string) - The username used to connect to the OpenStack service. If not specified, Packer will use the environment variable From a3863c3495db25eed06da1a2a5741f31dd9c8935 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 12 Jun 2015 15:23:05 -0400 Subject: [PATCH 83/85] builder/openstack: update floating IP messaging --- builder/openstack/step_allocate_ip.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/builder/openstack/step_allocate_ip.go b/builder/openstack/step_allocate_ip.go index 0ab9b3529..fc386082b 100644 --- a/builder/openstack/step_allocate_ip.go +++ b/builder/openstack/step_allocate_ip.go @@ -36,6 +36,8 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { if s.FloatingIp != "" { instanceIp.IP = s.FloatingIp } else if s.FloatingIpPool != "" { + ui.Say(fmt.Sprintf("Creating floating IP...")) + ui.Message(fmt.Sprintf("Pool: %s", s.FloatingIpPool)) newIp, err := floatingip.Create(client, floatingip.CreateOpts{ Pool: s.FloatingIpPool, }).Extract() @@ -47,22 +49,24 @@ func (s *StepAllocateIp) Run(state multistep.StateBag) multistep.StepAction { } instanceIp = *newIp - ui.Say(fmt.Sprintf("Created temporary floating IP %s...", instanceIp.IP)) + ui.Message(fmt.Sprintf("Created floating IP: %s", instanceIp.IP)) } if instanceIp.IP != "" { + ui.Say(fmt.Sprintf("Associating floating IP with server...")) + ui.Message(fmt.Sprintf("IP: %s", instanceIp.IP)) err := floatingip.Associate(client, server.ID, instanceIp.IP).ExtractErr() if err != nil { err := fmt.Errorf( - "Error associating floating IP %s with instance.", - instanceIp.IP) + "Error associating floating IP %s with instance: %s", + instanceIp.IP, err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - ui.Say(fmt.Sprintf( - "Added floating IP %s to instance...", instanceIp.IP)) + ui.Message(fmt.Sprintf( + "Added floating IP %s to instance!", instanceIp.IP)) } state.Put("access_ip", &instanceIp) From f398352996bc5e696ee17c47460099c3fcbd0ffb Mon Sep 17 00:00:00 2001 From: Mark Peek Date: Fri, 12 Jun 2015 14:00:59 -0700 Subject: [PATCH 84/85] Fix a bug where interpolation was broken in some builders --- builder/googlecompute/config.go | 2 +- builder/null/config.go | 2 +- builder/parallels/pvm/config.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/googlecompute/config.go b/builder/googlecompute/config.go index 743e4745c..69d223a75 100644 --- a/builder/googlecompute/config.go +++ b/builder/googlecompute/config.go @@ -47,7 +47,7 @@ type Config struct { func NewConfig(raws ...interface{}) (*Config, []string, error) { c := new(Config) - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ diff --git a/builder/null/config.go b/builder/null/config.go index 7665aec51..9ccc32282 100644 --- a/builder/null/config.go +++ b/builder/null/config.go @@ -21,7 +21,7 @@ type Config struct { func NewConfig(raws ...interface{}) (*Config, []string, error) { c := new(Config) - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ diff --git a/builder/parallels/pvm/config.go b/builder/parallels/pvm/config.go index c3fab4446..8f6d9a915 100644 --- a/builder/parallels/pvm/config.go +++ b/builder/parallels/pvm/config.go @@ -33,7 +33,7 @@ type Config struct { func NewConfig(raws ...interface{}) (*Config, []string, error) { c := new(Config) - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ From 16320372d64adacf1c172af0535fe56114d3b611 Mon Sep 17 00:00:00 2001 From: Mark Peek Date: Fri, 12 Jun 2015 14:02:09 -0700 Subject: [PATCH 85/85] Make some builder config usage more consistent with other builders --- builder/digitalocean/config.go | 6 +++--- builder/docker/config.go | 6 +++--- builder/virtualbox/ovf/config.go | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index a19dabda2..178b54049 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -42,10 +42,10 @@ type Config struct { } func NewConfig(raws ...interface{}) (*Config, []string, error) { - var c Config + c := new(Config) var md mapstructure.Metadata - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Metadata: &md, Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ @@ -142,5 +142,5 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { } common.ScrubConfig(c, c.APIToken) - return &c, nil, nil + return c, nil, nil } diff --git a/builder/docker/config.go b/builder/docker/config.go index 024b915af..d5801c8ba 100644 --- a/builder/docker/config.go +++ b/builder/docker/config.go @@ -31,10 +31,10 @@ type Config struct { } func NewConfig(raws ...interface{}) (*Config, []string, error) { - var c Config + c := new(Config) var md mapstructure.Metadata - err := config.Decode(&c, &config.DecodeOpts{ + err := config.Decode(c, &config.DecodeOpts{ Metadata: &md, Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ @@ -91,5 +91,5 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { return nil, nil, errs } - return &c, nil, nil + return c, nil, nil } diff --git a/builder/virtualbox/ovf/config.go b/builder/virtualbox/ovf/config.go index de60bcc7b..837c3f37a 100644 --- a/builder/virtualbox/ovf/config.go +++ b/builder/virtualbox/ovf/config.go @@ -40,8 +40,8 @@ type Config struct { } func NewConfig(raws ...interface{}) (*Config, []string, error) { - var c Config - err := config.Decode(&c, &config.DecodeOpts{ + c := new(Config) + err := config.Decode(c, &config.DecodeOpts{ Interpolate: true, InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ @@ -132,5 +132,5 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) { c.ImportFlags = append(c.ImportFlags, "--options", c.ImportOpts) } - return &c, warnings, nil + return c, warnings, nil }