diff --git a/builder/amazon/chroot/builder.go b/builder/amazon/chroot/builder.go new file mode 100644 index 000000000..d96f518f6 --- /dev/null +++ b/builder/amazon/chroot/builder.go @@ -0,0 +1,199 @@ +// The chroot package is able to create an Amazon AMI without requiring +// the launch of a new instance for every build. It does this by attaching +// and mounting the root volume of another AMI and chrooting into that +// directory. It then creates an AMI from that attached drive. +package chroot + +import ( + "errors" + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" + "github.com/mitchellh/packer/builder/common" + "github.com/mitchellh/packer/packer" + "log" + "runtime" + "text/template" +) + +// The unique ID for this builder +const BuilderId = "mitchellh.amazon.chroot" + +// Config is the configuration that is chained through the steps and +// settable from the template. +type Config struct { + common.PackerConfig `mapstructure:",squash"` + awscommon.AccessConfig `mapstructure:",squash"` + + AMIName string `mapstructure:"ami_name"` + ChrootMounts [][]string `mapstructure:"chroot_mounts"` + CopyFiles []string `mapstructure:"copy_files"` + DevicePath string `mapstructure:"device_path"` + MountCommand string `mapstructure:"mount_command"` + MountPath string `mapstructure:"mount_path"` + SourceAmi string `mapstructure:"source_ami"` + UnmountCommand string `mapstructure:"unmount_command"` +} + +type Builder struct { + config Config + runner multistep.Runner +} + +func (b *Builder) Prepare(raws ...interface{}) error { + md, err := common.DecodeConfig(&b.config, raws...) + if err != nil { + return err + } + + // Defaults + if b.config.ChrootMounts == nil { + b.config.ChrootMounts = make([][]string, 0) + } + + if b.config.CopyFiles == nil { + b.config.CopyFiles = make([]string, 0) + } + + if len(b.config.ChrootMounts) == 0 { + b.config.ChrootMounts = [][]string{ + []string{"proc", "proc", "/proc"}, + []string{"sysfs", "sysfs", "/sys"}, + []string{"bind", "/dev", "/dev"}, + []string{"devpts", "devpts", "/dev/pts"}, + []string{"binfmt_misc", "binfmt_misc", "/proc/sys/fs/binfmt_misc"}, + } + } + + if len(b.config.CopyFiles) == 0 { + b.config.CopyFiles = []string{"/etc/resolv.conf"} + } + + if b.config.MountCommand == "" { + b.config.MountCommand = "mount" + } + + if b.config.MountPath == "" { + b.config.MountPath = "packer-amazon-chroot-volumes/{{.Device}}" + } + + if b.config.UnmountCommand == "" { + b.config.UnmountCommand = "umount" + } + + // Accumulate any errors + errs := common.CheckUnusedConfig(md) + errs = packer.MultiErrorAppend(errs, b.config.AccessConfig.Prepare()...) + + if b.config.AMIName == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("ami_name must be specified")) + } else { + _, err = template.New("ami").Parse(b.config.AMIName) + if err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("Failed parsing ami_name: %s", err)) + } + } + + for _, mounts := range b.config.ChrootMounts { + if len(mounts) != 3 { + errs = packer.MultiErrorAppend( + errs, errors.New("Each chroot_mounts entry should be three elements.")) + break + } + } + + if b.config.SourceAmi == "" { + errs = packer.MultiErrorAppend(errs, errors.New("source_ami is required.")) + } + + if errs != nil && len(errs.Errors) > 0 { + return errs + } + + log.Printf("Config: %+v", b.config) + return nil +} + +func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { + if runtime.GOOS != "linux" { + return nil, errors.New("The amazon-chroot builder only works on Linux environments.") + } + + region, err := b.config.Region() + if err != nil { + return nil, err + } + + auth, err := b.config.AccessConfig.Auth() + if err != nil { + return nil, err + } + + ec2conn := ec2.New(auth, region) + + // Setup the state bag and initial state for the steps + state := make(map[string]interface{}) + state["config"] = &b.config + state["ec2"] = ec2conn + state["hook"] = hook + state["ui"] = ui + + // Build the steps + steps := []multistep.Step{ + &StepInstanceInfo{}, + &StepSourceAMIInfo{}, + &StepFlock{}, + &StepPrepareDevice{}, + &StepCreateVolume{}, + &StepAttachVolume{}, + &StepEarlyUnflock{}, + &StepMountDevice{}, + &StepMountExtra{}, + &StepCopyFiles{}, + &StepChrootProvision{}, + &StepEarlyCleanup{}, + &StepSnapshot{}, + &StepRegisterAMI{}, + } + + // Run! + if b.config.PackerDebug { + b.runner = &multistep.DebugRunner{ + Steps: steps, + PauseFn: common.MultistepDebugFn(ui), + } + } else { + b.runner = &multistep.BasicRunner{Steps: steps} + } + + b.runner.Run(state) + + // If there was an error, return that + if rawErr, ok := state["error"]; ok { + return nil, rawErr.(error) + } + + // If there are no AMIs, then just return + if _, ok := state["amis"]; !ok { + return nil, nil + } + + // Build the artifact and return it + artifact := &awscommon.Artifact{ + Amis: state["amis"].(map[string]string), + BuilderIdValue: BuilderId, + Conn: ec2conn, + } + + return artifact, nil +} + +func (b *Builder) Cancel() { + if b.runner != nil { + log.Println("Cancelling the step runner...") + b.runner.Cancel() + } +} diff --git a/builder/amazon/chroot/builder_test.go b/builder/amazon/chroot/builder_test.go new file mode 100644 index 000000000..2f13bae54 --- /dev/null +++ b/builder/amazon/chroot/builder_test.go @@ -0,0 +1,84 @@ +package chroot + +import ( + "github.com/mitchellh/packer/packer" + "testing" +) + +func testConfig() map[string]interface{} { + return map[string]interface{}{ + "ami_name": "foo", + "source_ami": "foo", + } +} + +func TestBuilder_ImplementsBuilder(t *testing.T) { + var raw interface{} + raw = &Builder{} + if _, ok := raw.(packer.Builder); !ok { + t.Fatalf("Builder should be a builder") + } +} + +func TestBuilderPrepare_AMIName(t *testing.T) { + var b Builder + config := testConfig() + + // Test good + config["ami_name"] = "foo" + err := b.Prepare(config) + if err != nil { + t.Fatalf("should not have error: %s", err) + } + + // Test bad + config["ami_name"] = "foo {{" + b = Builder{} + err = b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + // Test bad + delete(config, "ami_name") + b = Builder{} + err = b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } +} + +func TestBuilderPrepare_ChrootMounts(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["chroot_mounts"] = nil + err := b.Prepare(config) + if err != nil { + t.Errorf("err: %s", err) + } + + config["chroot_mounts"] = [][]string{ + []string{"bad"}, + } + err = b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } +} +func TestBuilderPrepare_SourceAmi(t *testing.T) { + b := &Builder{} + config := testConfig() + + config["source_ami"] = "" + err := b.Prepare(config) + if err == nil { + t.Fatal("should have error") + } + + config["source_ami"] = "foo" + err = b.Prepare(config) + if err != nil { + t.Errorf("err: %s", err) + } +} diff --git a/builder/amazon/chroot/cleanup.go b/builder/amazon/chroot/cleanup.go new file mode 100644 index 000000000..14c47aac3 --- /dev/null +++ b/builder/amazon/chroot/cleanup.go @@ -0,0 +1,6 @@ +package chroot + +// Cleanup is an interface that some steps implement for early cleanup. +type Cleanup interface { + CleanupFunc(map[string]interface{}) error +} diff --git a/builder/amazon/chroot/communicator.go b/builder/amazon/chroot/communicator.go new file mode 100644 index 000000000..e0bc5613f --- /dev/null +++ b/builder/amazon/chroot/communicator.go @@ -0,0 +1,87 @@ +package chroot + +import ( + "github.com/mitchellh/packer/packer" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "syscall" +) + +// Communicator is a special communicator that works by executing +// commands locally but within a chroot. +type Communicator struct { + Chroot string +} + +func (c *Communicator) Start(cmd *packer.RemoteCmd) error { + chrootCmdPath, err := exec.LookPath("chroot") + if err != nil { + return err + } + + localCmd := exec.Command(chrootCmdPath, c.Chroot, "/bin/sh", "-c", cmd.Command) + localCmd.Stdin = cmd.Stdin + localCmd.Stdout = cmd.Stdout + localCmd.Stderr = cmd.Stderr + log.Printf("Executing: %s %#v", localCmd.Path, localCmd.Args) + if err := localCmd.Start(); err != nil { + return err + } + + go func() { + exitStatus := 0 + if err := localCmd.Wait(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + exitStatus = 1 + + // There is no process-independent way to get the REAL + // exit status so we just try to go deeper. + if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { + exitStatus = status.ExitStatus() + } + } + } + + log.Printf( + "Chroot executation ended with '%d': '%s'", + exitStatus, cmd.Command) + cmd.SetExited(exitStatus) + }() + + return nil +} + +func (c *Communicator) Upload(dst string, r io.Reader) error { + dst = filepath.Join(c.Chroot, dst) + log.Printf("Uploading to chroot dir: %s", dst) + f, err := os.Create(dst) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.Copy(f, r); err != nil { + return err + } + + return nil +} + +func (c *Communicator) Download(src string, w io.Writer) error { + src = filepath.Join(c.Chroot, src) + log.Printf("Downloading from chroot dir: %s", src) + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.Copy(w, f); err != nil { + return err + } + + return nil +} diff --git a/builder/amazon/chroot/communicator_test.go b/builder/amazon/chroot/communicator_test.go new file mode 100644 index 000000000..56745a681 --- /dev/null +++ b/builder/amazon/chroot/communicator_test.go @@ -0,0 +1,14 @@ +package chroot + +import ( + "github.com/mitchellh/packer/packer" + "testing" +) + +func TestCommunicator_ImplementsCommunicator(t *testing.T) { + var raw interface{} + raw = &Communicator{} + if _, ok := raw.(packer.Communicator); !ok { + t.Fatalf("Communicator should be a communicator") + } +} diff --git a/builder/amazon/chroot/device.go b/builder/amazon/chroot/device.go new file mode 100644 index 000000000..ed5be6194 --- /dev/null +++ b/builder/amazon/chroot/device.go @@ -0,0 +1,61 @@ +package chroot + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// AvailableDevice finds an available device and returns it. Note that +// you should externally hold a flock or something in order to guarantee +// that this device is available across processes. +func AvailableDevice() (string, error) { + prefix, err := devicePrefix() + if err != nil { + return "", err + } + + letters := "fghijklmnop" + for _, letter := range letters { + for i := 1; i < 16; i++ { + device := fmt.Sprintf("/dev/%s%c%d", prefix, letter, i) + if _, err := os.Stat(device); err != nil { + return device, nil + } + } + } + + return "", errors.New("available device could not be found") +} + +// devicePrefix returns the prefix ("sd" or "xvd" or so on) of the devices +// on the system. +func devicePrefix() (string, error) { + available := []string{"sd", "xvd"} + + f, err := os.Open("/sys/block") + if err != nil { + return "", err + } + defer f.Close() + + dirs, err := f.Readdirnames(-1) + if dirs != nil && len(dirs) > 0 { + for _, dir := range dirs { + dirBase := filepath.Base(dir) + for _, prefix := range available { + if strings.HasPrefix(dirBase, prefix) { + return prefix, nil + } + } + } + } + + if err != nil { + return "", err + } + + return "", errors.New("device prefix could not be detected") +} diff --git a/builder/amazon/chroot/lockfile.go b/builder/amazon/chroot/lockfile.go new file mode 100644 index 000000000..1ba13e04b --- /dev/null +++ b/builder/amazon/chroot/lockfile.go @@ -0,0 +1,16 @@ +// +build windows + +package chroot + +import ( + "errors" + "os" +) + +func lockFile(*os.File) error { + return errors.New("not supported on Windows") +} + +func unlockFile(f *os.File) error { + return nil +} diff --git a/builder/amazon/chroot/lockfile_unix.go b/builder/amazon/chroot/lockfile_unix.go new file mode 100644 index 000000000..f84ea0d57 --- /dev/null +++ b/builder/amazon/chroot/lockfile_unix.go @@ -0,0 +1,32 @@ +// +build !windows + +package chroot + +import ( + "errors" + "os" + "syscall" +) + +// See: http://linux.die.net/include/sys/file.h +const LOCK_EX = 2 +const LOCK_NB = 4 +const LOCK_UN = 8 + +func lockFile(f *os.File) error { + err := syscall.Flock(int(f.Fd()), LOCK_EX|LOCK_NB) + if err != nil { + errno, ok := err.(syscall.Errno) + if ok && errno == syscall.EWOULDBLOCK { + return errors.New("file already locked") + } + + return err + } + + return nil +} + +func unlockFile(f *os.File) error { + return syscall.Flock(int(f.Fd()), LOCK_UN) +} diff --git a/builder/amazon/chroot/step_attach_volume.go b/builder/amazon/chroot/step_attach_volume.go new file mode 100644 index 000000000..051c0704c --- /dev/null +++ b/builder/amazon/chroot/step_attach_volume.go @@ -0,0 +1,129 @@ +package chroot + +import ( + "errors" + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" + "github.com/mitchellh/packer/packer" + "strings" +) + +// StepAttachVolume attaches the previously created volume to an +// available device location. +// +// Produces: +// device string - The location where the volume was attached. +// attach_cleanup CleanupFunc +type StepAttachVolume struct { + attached bool + volumeId string +} + +func (s *StepAttachVolume) Run(state map[string]interface{}) multistep.StepAction { + ec2conn := state["ec2"].(*ec2.EC2) + device := state["device"].(string) + instance := state["instance"].(*ec2.Instance) + ui := state["ui"].(packer.Ui) + volumeId := state["volume_id"].(string) + + // For the API call, it expects "sd" prefixed devices. + attachVolume := strings.Replace(device, "/xvd", "/sd", 1) + + ui.Say(fmt.Sprintf("Attaching the root volume to %s", attachVolume)) + _, err := ec2conn.AttachVolume(volumeId, instance.InstanceId, attachVolume) + if err != nil { + err := fmt.Errorf("Error attaching volume: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Mark that we attached it so we can detach it later + s.attached = true + s.volumeId = volumeId + + // Wait for the volume to become attached + stateChange := awscommon.StateChangeConf{ + Conn: ec2conn, + Pending: []string{"attaching"}, + StepState: state, + Target: "attached", + Refresh: func() (interface{}, string, error) { + resp, err := ec2conn.Volumes([]string{volumeId}, ec2.NewFilter()) + if err != nil { + return nil, "", err + } + + if len(resp.Volumes[0].Attachments) == 0 { + return nil, "", errors.New("No attachments on volume.") + } + + return nil, resp.Volumes[0].Attachments[0].Status, nil + }, + } + + _, err = awscommon.WaitForState(&stateChange) + if err != nil { + err := fmt.Errorf("Error waiting for volume: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + state["attach_cleanup"] = s + return multistep.ActionContinue +} + +func (s *StepAttachVolume) Cleanup(state map[string]interface{}) { + ui := state["ui"].(packer.Ui) + if err := s.CleanupFunc(state); err != nil { + ui.Error(err.Error()) + } +} + +func (s *StepAttachVolume) CleanupFunc(state map[string]interface{}) error { + if !s.attached { + return nil + } + + ec2conn := state["ec2"].(*ec2.EC2) + ui := state["ui"].(packer.Ui) + + ui.Say("Detaching EBS volume...") + _, err := ec2conn.DetachVolume(s.volumeId) + if err != nil { + return fmt.Errorf("Error detaching EBS volume: %s", err) + } + + s.attached = false + + // Wait for the volume to detach + stateChange := awscommon.StateChangeConf{ + Conn: ec2conn, + Pending: []string{"attaching", "attached", "detaching"}, + StepState: state, + Target: "detached", + Refresh: func() (interface{}, string, error) { + resp, err := ec2conn.Volumes([]string{s.volumeId}, ec2.NewFilter()) + if err != nil { + return nil, "", err + } + + state := "detached" + if len(resp.Volumes[0].Attachments) > 0 { + state = resp.Volumes[0].Attachments[0].Status + } + + return nil, state, nil + }, + } + + _, err = awscommon.WaitForState(&stateChange) + if err != nil { + return fmt.Errorf("Error waiting for volume: %s", err) + } + + return nil +} diff --git a/builder/amazon/chroot/step_attach_volume_test.go b/builder/amazon/chroot/step_attach_volume_test.go new file mode 100644 index 000000000..63f629b77 --- /dev/null +++ b/builder/amazon/chroot/step_attach_volume_test.go @@ -0,0 +1,11 @@ +package chroot + +import "testing" + +func TestAttachVolumeCleanupFunc_ImplementsCleanupFunc(t *testing.T) { + var raw interface{} + raw = new(StepAttachVolume) + if _, ok := raw.(Cleanup); !ok { + t.Fatalf("cleanup func should be a CleanupFunc") + } +} diff --git a/builder/amazon/chroot/step_chroot_provision.go b/builder/amazon/chroot/step_chroot_provision.go new file mode 100644 index 000000000..a295f27bf --- /dev/null +++ b/builder/amazon/chroot/step_chroot_provision.go @@ -0,0 +1,34 @@ +package chroot + +import ( + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" +) + +// StepChrootProvision provisions the instance within a chroot. +type StepChrootProvision struct { + mounts []string +} + +func (s *StepChrootProvision) Run(state map[string]interface{}) multistep.StepAction { + hook := state["hook"].(packer.Hook) + mountPath := state["mount_path"].(string) + ui := state["ui"].(packer.Ui) + + // Create our communicator + comm := &Communicator{ + Chroot: mountPath, + } + + // Provision + log.Println("Running the provision hook") + if err := hook.Run(packer.HookProvision, ui, comm, nil); err != nil { + state["error"] = err + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *StepChrootProvision) Cleanup(state map[string]interface{}) {} diff --git a/builder/amazon/chroot/step_copy_files.go b/builder/amazon/chroot/step_copy_files.go new file mode 100644 index 000000000..45ec09db3 --- /dev/null +++ b/builder/amazon/chroot/step_copy_files.go @@ -0,0 +1,107 @@ +package chroot + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "io" + "log" + "os" + "path/filepath" +) + +// StepCopyFiles copies some files from the host into the chroot environment. +// +// Produces: +// copy_files_cleanup CleanupFunc - A function to clean up the copied files +// early. +type StepCopyFiles struct { + files []string +} + +func (s *StepCopyFiles) Run(state map[string]interface{}) multistep.StepAction { + config := state["config"].(*Config) + mountPath := state["mount_path"].(string) + ui := state["ui"].(packer.Ui) + + s.files = make([]string, 0, len(config.CopyFiles)) + if len(config.CopyFiles) > 0 { + ui.Say("Copying files from host to chroot...") + for _, path := range config.CopyFiles { + ui.Message(path) + chrootPath := filepath.Join(mountPath, path) + log.Printf("Copying '%s' to '%s'", path, chrootPath) + + if err := s.copySingle(chrootPath, path); err != nil { + err := fmt.Errorf("Error copying file: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + s.files = append(s.files, chrootPath) + } + } + + state["copy_files_cleanup"] = s + return multistep.ActionContinue +} + +func (s *StepCopyFiles) Cleanup(state map[string]interface{}) { + ui := state["ui"].(packer.Ui) + if err := s.CleanupFunc(state); err != nil { + ui.Error(err.Error()) + } +} + +func (s *StepCopyFiles) CleanupFunc(map[string]interface{}) error { + if s.files != nil { + for _, file := range s.files { + log.Printf("Removing: %s", file) + if err := os.Remove(file); err != nil { + return err + } + } + } + + s.files = nil + return nil +} + +func (s *StepCopyFiles) copySingle(dst, src string) error { + // Stat the src file so we can copy the mode later + srcInfo, err := os.Stat(src) + if err != nil { + return err + } + + // Remove any existing destination file + if err := os.Remove(dst); err != nil { + return err + } + + // Copy the files + srcF, err := os.Open(src) + if err != nil { + return err + } + defer srcF.Close() + + dstF, err := os.Create(dst) + if err != nil { + return err + } + defer dstF.Close() + + if _, err := io.Copy(dstF, srcF); err != nil { + return err + } + dstF.Close() + + // Match the mode + if err := os.Chmod(dst, srcInfo.Mode()); err != nil { + return err + } + + return nil +} diff --git a/builder/amazon/chroot/step_copy_files_test.go b/builder/amazon/chroot/step_copy_files_test.go new file mode 100644 index 000000000..281613e6f --- /dev/null +++ b/builder/amazon/chroot/step_copy_files_test.go @@ -0,0 +1,11 @@ +package chroot + +import "testing" + +func TestCopyFilesCleanupFunc_ImplementsCleanupFunc(t *testing.T) { + var raw interface{} + raw = new(StepCopyFiles) + if _, ok := raw.(Cleanup); !ok { + t.Fatalf("cleanup func should be a CleanupFunc") + } +} diff --git a/builder/amazon/chroot/step_create_volume.go b/builder/amazon/chroot/step_create_volume.go new file mode 100644 index 000000000..d0e247989 --- /dev/null +++ b/builder/amazon/chroot/step_create_volume.go @@ -0,0 +1,107 @@ +package chroot + +import ( + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" + "github.com/mitchellh/packer/packer" + "log" +) + +// StepCreateVolume creates a new volume from the snapshot of the root +// device of the AMI. +// +// Produces: +// volume_id string - The ID of the created volume +type StepCreateVolume struct { + volumeId string +} + +func (s *StepCreateVolume) Run(state map[string]interface{}) multistep.StepAction { + ec2conn := state["ec2"].(*ec2.EC2) + image := state["source_image"].(*ec2.Image) + instance := state["instance"].(*ec2.Instance) + ui := state["ui"].(packer.Ui) + + // Determine the root device snapshot + log.Printf("Searching for root device of the image (%s)", image.RootDeviceName) + var rootDevice *ec2.BlockDeviceMapping + for _, device := range image.BlockDevices { + if device.DeviceName == image.RootDeviceName { + rootDevice = &device + break + } + } + + if rootDevice == nil { + err := fmt.Errorf("Couldn't find root device!") + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + ui.Say("Creating the root volume...") + createVolume := &ec2.CreateVolume{ + AvailZone: instance.AvailZone, + Size: rootDevice.VolumeSize, + SnapshotId: rootDevice.SnapshotId, + VolumeType: rootDevice.VolumeType, + IOPS: rootDevice.IOPS, + } + log.Printf("Create args: %#v", createVolume) + + createVolumeResp, err := ec2conn.CreateVolume(createVolume) + if err != nil { + err := fmt.Errorf("Error creating root volume: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Set the volume ID so we remember to delete it later + s.volumeId = createVolumeResp.VolumeId + log.Printf("Volume ID: %s", s.volumeId) + + // Wait for the volume to become ready + stateChange := awscommon.StateChangeConf{ + Conn: ec2conn, + Pending: []string{"creating"}, + StepState: state, + Target: "available", + Refresh: func() (interface{}, string, error) { + resp, err := ec2conn.Volumes([]string{s.volumeId}, ec2.NewFilter()) + if err != nil { + return nil, "", err + } + + return nil, resp.Volumes[0].Status, nil + }, + } + + _, err = awscommon.WaitForState(&stateChange) + if err != nil { + err := fmt.Errorf("Error waiting for volume: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + state["volume_id"] = s.volumeId + return multistep.ActionContinue +} + +func (s *StepCreateVolume) Cleanup(state map[string]interface{}) { + if s.volumeId == "" { + return + } + + ec2conn := state["ec2"].(*ec2.EC2) + ui := state["ui"].(packer.Ui) + + ui.Say("Deleting the created EBS volume...") + _, err := ec2conn.DeleteVolume(s.volumeId) + if err != nil { + ui.Error(fmt.Sprintf("Error deleting EBS volume: %s", err)) + } +} diff --git a/builder/amazon/chroot/step_early_cleanup.go b/builder/amazon/chroot/step_early_cleanup.go new file mode 100644 index 000000000..01ccb48d7 --- /dev/null +++ b/builder/amazon/chroot/step_early_cleanup.go @@ -0,0 +1,37 @@ +package chroot + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" +) + +// StepEarlyCleanup performs some of the cleanup steps early in order to +// prepare for snapshotting and creating an AMI. +type StepEarlyCleanup struct{} + +func (s *StepEarlyCleanup) Run(state map[string]interface{}) multistep.StepAction { + ui := state["ui"].(packer.Ui) + cleanupKeys := []string{ + "copy_files_cleanup", + "mount_extra_cleanup", + "mount_device_cleanup", + "attach_cleanup", + } + + for _, key := range cleanupKeys { + c := state[key].(Cleanup) + log.Printf("Running cleanup func: %s", key) + if err := c.CleanupFunc(state); err != nil { + err := fmt.Errorf("Error cleaning up: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + } + + return multistep.ActionContinue +} + +func (s *StepEarlyCleanup) Cleanup(state map[string]interface{}) {} diff --git a/builder/amazon/chroot/step_early_unflock.go b/builder/amazon/chroot/step_early_unflock.go new file mode 100644 index 000000000..d1c09008e --- /dev/null +++ b/builder/amazon/chroot/step_early_unflock.go @@ -0,0 +1,28 @@ +package chroot + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" +) + +// StepEarlyUnflock unlocks the flock. +type StepEarlyUnflock struct{} + +func (s *StepEarlyUnflock) Run(state map[string]interface{}) multistep.StepAction { + cleanup := state["flock_cleanup"].(Cleanup) + ui := state["ui"].(packer.Ui) + + log.Println("Unlocking file lock...") + if err := cleanup.CleanupFunc(state); err != nil { + err := fmt.Errorf("Error unlocking file lock: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *StepEarlyUnflock) Cleanup(state map[string]interface{}) {} diff --git a/builder/amazon/chroot/step_flock.go b/builder/amazon/chroot/step_flock.go new file mode 100644 index 000000000..47d168d4c --- /dev/null +++ b/builder/amazon/chroot/step_flock.go @@ -0,0 +1,72 @@ +package chroot + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" + "os" + "path/filepath" +) + +// StepFlock provisions the instance within a chroot. +// +// Produces: +// flock_cleanup Cleanup - To perform early cleanup +type StepFlock struct { + fh *os.File +} + +func (s *StepFlock) Run(state map[string]interface{}) multistep.StepAction { + ui := state["ui"].(packer.Ui) + + lockfile := "/var/lock/packer-chroot/lock" + if err := os.MkdirAll(filepath.Dir(lockfile), 0755); err != nil { + err := fmt.Errorf("Error creating lock: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + log.Printf("Obtaining lock: %s", lockfile) + f, err := os.Create(lockfile) + if err != nil { + err := fmt.Errorf("Error creating lock: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // LOCK! + if err := lockFile(f); err != nil { + err := fmt.Errorf("Error creating lock: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Set the file handle, we can't close it because we need to hold + // the lock. + s.fh = f + + state["flock_cleanup"] = s + return multistep.ActionContinue +} + +func (s *StepFlock) Cleanup(state map[string]interface{}) { + s.CleanupFunc(state) +} + +func (s *StepFlock) CleanupFunc(state map[string]interface{}) error { + if s.fh == nil { + return nil + } + + log.Printf("Unlocking: %s", s.fh.Name()) + if err := unlockFile(s.fh); err != nil { + return err + } + + s.fh = nil + return nil +} diff --git a/builder/amazon/chroot/step_flock_test.go b/builder/amazon/chroot/step_flock_test.go new file mode 100644 index 000000000..a50cbd93d --- /dev/null +++ b/builder/amazon/chroot/step_flock_test.go @@ -0,0 +1,11 @@ +package chroot + +import "testing" + +func TestFlockCleanupFunc_ImplementsCleanupFunc(t *testing.T) { + var raw interface{} + raw = new(StepFlock) + if _, ok := raw.(Cleanup); !ok { + t.Fatalf("cleanup func should be a CleanupFunc") + } +} diff --git a/builder/amazon/chroot/step_instance_info.go b/builder/amazon/chroot/step_instance_info.go new file mode 100644 index 000000000..53080e171 --- /dev/null +++ b/builder/amazon/chroot/step_instance_info.go @@ -0,0 +1,57 @@ +package chroot + +import ( + "fmt" + "github.com/mitchellh/goamz/aws" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" +) + +// StepInstanceInfo verifies that this builder is running on an EC2 instance. +type StepInstanceInfo struct{} + +func (s *StepInstanceInfo) Run(state map[string]interface{}) multistep.StepAction { + ec2conn := state["ec2"].(*ec2.EC2) + ui := state["ui"].(packer.Ui) + + // Get our own instance ID + ui.Say("Gathering information about this EC2 instance...") + instanceIdBytes, err := aws.GetMetaData("instance-id") + if err != nil { + log.Printf("Error: %s", err) + err := fmt.Errorf( + "Error retrieving the ID of the instance Packer is running on.\n" + + "Please verify Packer is running on a proper AWS EC2 instance.") + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + instanceId := string(instanceIdBytes) + log.Printf("Instance ID: %s", instanceId) + + // Query the entire instance metadata + instancesResp, err := ec2conn.Instances([]string{instanceId}, ec2.NewFilter()) + if err != nil { + err := fmt.Errorf("Error getting instance data: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if len(instancesResp.Reservations) == 0 { + err := fmt.Errorf("Error getting instance data: no instance found.") + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + instance := &instancesResp.Reservations[0].Instances[0] + state["instance"] = instance + + return multistep.ActionContinue +} + +func (s *StepInstanceInfo) Cleanup(map[string]interface{}) {} diff --git a/builder/amazon/chroot/step_mount_device.go b/builder/amazon/chroot/step_mount_device.go new file mode 100644 index 000000000..8d6815427 --- /dev/null +++ b/builder/amazon/chroot/step_mount_device.go @@ -0,0 +1,103 @@ +package chroot + +import ( + "bytes" + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" + "os" + "os/exec" + "path/filepath" + "text/template" +) + +type mountPathData struct { + Device string +} + +// StepMountDevice mounts the attached device. +// +// Produces: +// mount_path string - The location where the volume was mounted. +// mount_device_cleanup CleanupFunc - To perform early cleanup +type StepMountDevice struct { + mountPath string +} + +func (s *StepMountDevice) Run(state map[string]interface{}) multistep.StepAction { + config := state["config"].(*Config) + ui := state["ui"].(packer.Ui) + device := state["device"].(string) + + mountPathRaw := new(bytes.Buffer) + t := template.Must(template.New("mountPath").Parse(config.MountPath)) + t.Execute(mountPathRaw, &mountPathData{ + Device: filepath.Base(device), + }) + + var err error + mountPath := mountPathRaw.String() + mountPath, err = filepath.Abs(mountPath) + if err != nil { + err := fmt.Errorf("Error preparing mount directory: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + log.Printf("Mount path: %s", mountPath) + + if err := os.MkdirAll(mountPath, 0755); err != nil { + err := fmt.Errorf("Error creating mount directory: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + ui.Say("Mounting the root device...") + stderr := new(bytes.Buffer) + mountCommand := fmt.Sprintf("%s %s %s", config.MountCommand, device, mountPath) + cmd := exec.Command("/bin/sh", "-c", mountCommand) + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + err := fmt.Errorf( + "Error mounting root volume: %s\nStderr: %s", err, stderr.String()) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Set the mount path so we remember to unmount it later + s.mountPath = mountPath + state["mount_path"] = s.mountPath + state["mount_device_cleanup"] = s + + return multistep.ActionContinue +} + +func (s *StepMountDevice) Cleanup(state map[string]interface{}) { + ui := state["ui"].(packer.Ui) + if err := s.CleanupFunc(state); err != nil { + ui.Error(err.Error()) + } +} + +func (s *StepMountDevice) CleanupFunc(state map[string]interface{}) error { + if s.mountPath == "" { + return nil + } + + config := state["config"].(*Config) + ui := state["ui"].(packer.Ui) + ui.Say("Unmounting the root device...") + + unmountCommand := fmt.Sprintf("%s %s", config.UnmountCommand, s.mountPath) + cmd := exec.Command("/bin/sh", "-c", unmountCommand) + if err := cmd.Run(); err != nil { + return fmt.Errorf("Error unmounting root device: %s", err) + } + + s.mountPath = "" + return nil +} diff --git a/builder/amazon/chroot/step_mount_device_test.go b/builder/amazon/chroot/step_mount_device_test.go new file mode 100644 index 000000000..2eeb850eb --- /dev/null +++ b/builder/amazon/chroot/step_mount_device_test.go @@ -0,0 +1,11 @@ +package chroot + +import "testing" + +func TestMountDeviceCleanupFunc_ImplementsCleanupFunc(t *testing.T) { + var raw interface{} + raw = new(StepMountDevice) + if _, ok := raw.(Cleanup); !ok { + t.Fatalf("cleanup func should be a CleanupFunc") + } +} diff --git a/builder/amazon/chroot/step_mount_extra.go b/builder/amazon/chroot/step_mount_extra.go new file mode 100644 index 000000000..6a306fdf9 --- /dev/null +++ b/builder/amazon/chroot/step_mount_extra.go @@ -0,0 +1,100 @@ +package chroot + +import ( + "bytes" + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "os" + "os/exec" +) + +// StepMountExtra mounts the attached device. +// +// Produces: +// mount_extra_cleanup CleanupFunc - To perform early cleanup +type StepMountExtra struct { + mounts []string +} + +func (s *StepMountExtra) Run(state map[string]interface{}) multistep.StepAction { + config := state["config"].(*Config) + mountPath := state["mount_path"].(string) + ui := state["ui"].(packer.Ui) + + s.mounts = make([]string, 0, len(config.ChrootMounts)) + + ui.Say("Mounting additional paths within the chroot...") + for _, mountInfo := range config.ChrootMounts { + innerPath := mountPath + mountInfo[2] + + if err := os.MkdirAll(innerPath, 0755); err != nil { + err := fmt.Errorf("Error creating mount directory: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + flags := "-t " + mountInfo[0] + if mountInfo[0] == "bind" { + flags = "--bind" + } + + ui.Message(fmt.Sprintf("Mounting: %s", mountInfo[2])) + stderr := new(bytes.Buffer) + mountCommand := fmt.Sprintf( + "%s %s %s %s", + config.MountCommand, + flags, + mountInfo[1], + innerPath) + cmd := exec.Command("/bin/sh", "-c", mountCommand) + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + err := fmt.Errorf( + "Error mounting: %s\nStderr: %s", err, stderr.String()) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + s.mounts = append(s.mounts, innerPath) + } + + state["mount_extra_cleanup"] = s + return multistep.ActionContinue +} + +func (s *StepMountExtra) Cleanup(state map[string]interface{}) { + ui := state["ui"].(packer.Ui) + + if err := s.CleanupFunc(state); err != nil { + ui.Error(err.Error()) + return + } +} + +func (s *StepMountExtra) CleanupFunc(state map[string]interface{}) error { + if s.mounts == nil { + return nil + } + + config := state["config"].(*Config) + for len(s.mounts) > 0 { + var path string + lastIndex := len(s.mounts) - 1 + path, s.mounts = s.mounts[lastIndex], s.mounts[:lastIndex] + unmountCommand := fmt.Sprintf("%s %s", config.UnmountCommand, path) + + stderr := new(bytes.Buffer) + cmd := exec.Command("/bin/sh", "-c", unmountCommand) + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf( + "Error unmounting device: %s\nStderr: %s", err, stderr.String()) + } + } + + s.mounts = nil + return nil +} diff --git a/builder/amazon/chroot/step_mount_extra_test.go b/builder/amazon/chroot/step_mount_extra_test.go new file mode 100644 index 000000000..d53cc7056 --- /dev/null +++ b/builder/amazon/chroot/step_mount_extra_test.go @@ -0,0 +1,11 @@ +package chroot + +import "testing" + +func TestMountExtraCleanupFunc_ImplementsCleanupFunc(t *testing.T) { + var raw interface{} + raw = new(StepMountExtra) + if _, ok := raw.(Cleanup); !ok { + t.Fatalf("cleanup func should be a CleanupFunc") + } +} diff --git a/builder/amazon/chroot/step_prepare_device.go b/builder/amazon/chroot/step_prepare_device.go new file mode 100644 index 000000000..114c0ce9d --- /dev/null +++ b/builder/amazon/chroot/step_prepare_device.go @@ -0,0 +1,45 @@ +package chroot + +import ( + "fmt" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" + "os" +) + +// StepPrepareDevice finds an available device and sets it. +type StepPrepareDevice struct { + mounts []string +} + +func (s *StepPrepareDevice) Run(state map[string]interface{}) multistep.StepAction { + config := state["config"].(*Config) + ui := state["ui"].(packer.Ui) + + device := config.DevicePath + if device == "" { + var err error + log.Println("Device path not specified, searching for available device...") + device, err = AvailableDevice() + if err != nil { + err := fmt.Errorf("Error finding available device: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + } + + if _, err := os.Stat(device); err == nil { + err := fmt.Errorf("Device is in use: %s", device) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + log.Printf("Device: %s", device) + state["device"] = device + return multistep.ActionContinue +} + +func (s *StepPrepareDevice) Cleanup(state map[string]interface{}) {} diff --git a/builder/amazon/chroot/step_register_ami.go b/builder/amazon/chroot/step_register_ami.go new file mode 100644 index 000000000..1c08bda61 --- /dev/null +++ b/builder/amazon/chroot/step_register_ami.go @@ -0,0 +1,84 @@ +package chroot + +import ( + "bytes" + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" + "github.com/mitchellh/packer/packer" + "strconv" + "text/template" + "time" +) + +type amiNameData struct { + CreateTime string +} + +// StepRegisterAMI creates the AMI. +type StepRegisterAMI struct{} + +func (s *StepRegisterAMI) Run(state map[string]interface{}) multistep.StepAction { + config := state["config"].(*Config) + ec2conn := state["ec2"].(*ec2.EC2) + image := state["source_image"].(*ec2.Image) + snapshotId := state["snapshot_id"].(string) + ui := state["ui"].(packer.Ui) + + // Parse the name of the AMI + amiNameBuf := new(bytes.Buffer) + tData := amiNameData{ + strconv.FormatInt(time.Now().UTC().Unix(), 10), + } + + t := template.Must(template.New("ami").Parse(config.AMIName)) + t.Execute(amiNameBuf, tData) + amiName := amiNameBuf.String() + + ui.Say("Registering the AMI...") + blockDevices := make([]ec2.BlockDeviceMapping, len(image.BlockDevices)) + for i, device := range image.BlockDevices { + newDevice := device + if newDevice.DeviceName == image.RootDeviceName { + newDevice.SnapshotId = snapshotId + } + + blockDevices[i] = newDevice + } + + registerOpts := &ec2.RegisterImage{ + Name: amiName, + Architecture: image.Architecture, + KernelId: image.KernelId, + RamdiskId: image.RamdiskId, + RootDeviceName: image.RootDeviceName, + BlockDevices: blockDevices, + } + + registerResp, err := ec2conn.RegisterImage(registerOpts) + if err != nil { + state["error"] = fmt.Errorf("Error registering AMI: %s", err) + ui.Error(state["error"].(error).Error()) + return multistep.ActionHalt + } + + // Set the AMI ID in the state + ui.Say(fmt.Sprintf("AMI: %s", registerResp.ImageId)) + amis := make(map[string]string) + amis[ec2conn.Region.Name] = registerResp.ImageId + state["amis"] = amis + + // Wait for the image to become ready + ui.Say("Waiting for AMI to become ready...") + if err := awscommon.WaitForAMI(ec2conn, registerResp.ImageId); err != nil { + err := fmt.Errorf("Error waiting for AMI: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + return multistep.ActionContinue +} + +func (s *StepRegisterAMI) Cleanup(state map[string]interface{}) {} diff --git a/builder/amazon/chroot/step_snapshot.go b/builder/amazon/chroot/step_snapshot.go new file mode 100644 index 000000000..837543ec8 --- /dev/null +++ b/builder/amazon/chroot/step_snapshot.go @@ -0,0 +1,87 @@ +package chroot + +import ( + "errors" + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + awscommon "github.com/mitchellh/packer/builder/amazon/common" + "github.com/mitchellh/packer/packer" +) + +// StepSnapshot creates a snapshot of the created volume. +// +// Produces: +// snapshot_id string - ID of the created snapshot +type StepSnapshot struct { + snapshotId string +} + +func (s *StepSnapshot) Run(state map[string]interface{}) multistep.StepAction { + ec2conn := state["ec2"].(*ec2.EC2) + ui := state["ui"].(packer.Ui) + volumeId := state["volume_id"].(string) + + ui.Say("Creating snapshot...") + createSnapResp, err := ec2conn.CreateSnapshot(volumeId, "") + if err != nil { + err := fmt.Errorf("Error creating snapshot: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + // Set the snapshot ID so we can delete it later + s.snapshotId = createSnapResp.Id + ui.Message(fmt.Sprintf("Snapshot ID: %s", s.snapshotId)) + + // Wait for the snapshot to be ready + stateChange := awscommon.StateChangeConf{ + Conn: ec2conn, + Pending: []string{"pending"}, + StepState: state, + Target: "completed", + Refresh: func() (interface{}, string, error) { + resp, err := ec2conn.Snapshots([]string{s.snapshotId}, ec2.NewFilter()) + if err != nil { + return nil, "", err + } + + if len(resp.Snapshots) == 0 { + return nil, "", errors.New("No snapshots found.") + } + + return nil, resp.Snapshots[0].Status, nil + }, + } + + _, err = awscommon.WaitForState(&stateChange) + if err != nil { + err := fmt.Errorf("Error waiting for snapshot: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + state["snapshot_id"] = s.snapshotId + return multistep.ActionContinue +} + +func (s *StepSnapshot) Cleanup(state map[string]interface{}) { + if s.snapshotId == "" { + return + } + + _, cancelled := state[multistep.StateCancelled] + _, halted := state[multistep.StateHalted] + + if cancelled || halted { + ec2conn := state["ec2"].(*ec2.EC2) + ui := state["ui"].(packer.Ui) + ui.Say("Removing snapshot since we cancelled or halted...") + _, err := ec2conn.DeleteSnapshots([]string{s.snapshotId}) + if err != nil { + ui.Error(fmt.Sprintf("Error: %s", err)) + } + } +} diff --git a/builder/amazon/chroot/step_source_ami_info.go b/builder/amazon/chroot/step_source_ami_info.go new file mode 100644 index 000000000..28f47149b --- /dev/null +++ b/builder/amazon/chroot/step_source_ami_info.go @@ -0,0 +1,52 @@ +package chroot + +import ( + "fmt" + "github.com/mitchellh/goamz/ec2" + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" +) + +// StepSourceAMIInfo extracts critical information from the source AMI +// that is used throughout the AMI creation process. +// +// Produces: +// source_image *ec2.Image - the source AMI info +type StepSourceAMIInfo struct{} + +func (s *StepSourceAMIInfo) Run(state map[string]interface{}) multistep.StepAction { + config := state["config"].(*Config) + ec2conn := state["ec2"].(*ec2.EC2) + ui := state["ui"].(packer.Ui) + + ui.Say("Inspecting the source AMI...") + imageResp, err := ec2conn.Images([]string{config.SourceAmi}, ec2.NewFilter()) + if err != nil { + err := fmt.Errorf("Error querying AMI: %s", err) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if len(imageResp.Images) == 0 { + err := fmt.Errorf("Source AMI '%s' was not found!", config.SourceAmi) + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + image := &imageResp.Images[0] + + // It must be EBS-backed otherwise the build won't work + if image.RootDeviceType != "ebs" { + err := fmt.Errorf("The root device of the source AMI must be EBS-backed.") + state["error"] = err + ui.Error(err.Error()) + return multistep.ActionHalt + } + + state["source_image"] = image + return multistep.ActionContinue +} + +func (s *StepSourceAMIInfo) Cleanup(map[string]interface{}) {} diff --git a/builder/amazon/common/access_config.go b/builder/amazon/common/access_config.go index 736d2d190..e78c15fad 100644 --- a/builder/amazon/common/access_config.go +++ b/builder/amazon/common/access_config.go @@ -1,13 +1,17 @@ package common import ( + "fmt" "github.com/mitchellh/goamz/aws" + "strings" + "unicode" ) // AccessConfig is for common configuration related to AWS access type AccessConfig struct { AccessKey string `mapstructure:"access_key"` SecretKey string `mapstructure:"secret_key"` + RawRegion string `mapstructure:"region"` } // Auth returns a valid aws.Auth object for access to AWS services, or @@ -16,6 +20,28 @@ func (c *AccessConfig) Auth() (aws.Auth, error) { return aws.GetAuth(c.AccessKey, c.SecretKey) } +// Region returns the aws.Region object for access to AWS services, requesting +// the region from the instance metadata if possible. +func (c *AccessConfig) Region() (aws.Region, error) { + if c.RawRegion != "" { + return aws.Regions[c.RawRegion], nil + } + + md, err := aws.GetMetaData("placement/availability-zone") + if err != nil { + return aws.Region{}, err + } + + region := strings.TrimRightFunc(string(md), unicode.IsLetter) + return aws.Regions[region], nil +} + func (c *AccessConfig) Prepare() []error { + if c.RawRegion != "" { + if _, ok := aws.Regions[c.RawRegion]; !ok { + return []error{fmt.Errorf("Unknown region: %s", c.RawRegion)} + } + } + return nil } diff --git a/builder/amazon/common/access_config_test.go b/builder/amazon/common/access_config_test.go new file mode 100644 index 000000000..cfb9e07f1 --- /dev/null +++ b/builder/amazon/common/access_config_test.go @@ -0,0 +1,27 @@ +package common + +import ( + "testing" +) + +func testAccessConfig() *AccessConfig { + return &AccessConfig{} +} + +func TestAccessConfigPrepare_Region(t *testing.T) { + c := testAccessConfig() + c.RawRegion = "" + if err := c.Prepare(); err != nil { + t.Fatalf("shouldn't have err: %s", err) + } + + c.RawRegion = "us-east-12" + if err := c.Prepare(); err == nil { + t.Fatal("should have error") + } + + c.RawRegion = "us-east-1" + if err := c.Prepare(); err != nil { + t.Fatalf("shouldn't have err: %s", err) + } +} diff --git a/builder/amazon/common/instance.go b/builder/amazon/common/instance.go index 442ed597c..1ea763e54 100644 --- a/builder/amazon/common/instance.go +++ b/builder/amazon/common/instance.go @@ -11,17 +11,38 @@ import ( type StateChangeConf struct { Conn *ec2.EC2 - Instance *ec2.Instance Pending []string + Refresh func() (interface{}, string, error) StepState map[string]interface{} Target string } -func WaitForState(conf *StateChangeConf) (i *ec2.Instance, err error) { - log.Printf("Waiting for instance state to become: %s", conf.Target) +func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) func() (interface{}, string, error) { + return func() (interface{}, string, error) { + resp, err := conn.Instances([]string{i.InstanceId}, ec2.NewFilter()) + if err != nil { + return nil, "", err + } + + i = &resp.Reservations[0].Instances[0] + return i, i.State.Name, nil + } +} + +func WaitForState(conf *StateChangeConf) (i interface{}, err error) { + log.Printf("Waiting for state to become: %s", conf.Target) + + for { + var currentState string + i, currentState, err = conf.Refresh() + if err != nil { + return + } + + if currentState == conf.Target { + return + } - i = conf.Instance - for i.State.Name != conf.Target { if conf.StepState != nil { if _, ok := conf.StepState[multistep.StateCancelled]; ok { return nil, errors.New("interrupted") @@ -30,24 +51,17 @@ func WaitForState(conf *StateChangeConf) (i *ec2.Instance, err error) { found := false for _, allowed := range conf.Pending { - if i.State.Name == allowed { + if currentState == allowed { found = true break } } if !found { - fmt.Errorf("unexpected state '%s', wanted target '%s'", i.State.Name, conf.Target) + fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target) return } - var resp *ec2.InstancesResp - resp, err = conf.Conn.Instances([]string{i.InstanceId}, ec2.NewFilter()) - if err != nil { - return - } - - i = &resp.Reservations[0].Instances[0] time.Sleep(2 * time.Second) } diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index bd2186d93..321378cc9 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -3,14 +3,12 @@ package common import ( "errors" "fmt" - "github.com/mitchellh/goamz/aws" "time" ) // RunConfig contains configuration for running an instance from a source // AMI and details on how to access that launched image. type RunConfig struct { - Region string SourceAmi string `mapstructure:"source_ami"` InstanceType string `mapstructure:"instance_type"` RawSSHTimeout string `mapstructure:"ssh_timeout"` @@ -45,12 +43,6 @@ func (c *RunConfig) Prepare() []error { errs = append(errs, errors.New("An instance_type must be specified")) } - if c.Region == "" { - errs = append(errs, errors.New("A region must be specified")) - } else if _, ok := aws.Regions[c.Region]; !ok { - errs = append(errs, fmt.Errorf("Unknown region: %s", c.Region)) - } - if c.SSHUsername == "" { errs = append(errs, errors.New("An ssh_username must be specified")) } diff --git a/builder/amazon/common/run_config_test.go b/builder/amazon/common/run_config_test.go index 0c0d40680..b34b4c614 100644 --- a/builder/amazon/common/run_config_test.go +++ b/builder/amazon/common/run_config_test.go @@ -16,7 +16,6 @@ func init() { func testConfig() *RunConfig { return &RunConfig{ - Region: "us-east-1", SourceAmi: "abcd", InstanceType: "m1.small", SSHUsername: "root", @@ -39,24 +38,6 @@ func TestRunConfigPrepare_InstanceType(t *testing.T) { } } -func TestRunConfigPrepare_Region(t *testing.T) { - c := testConfig() - c.Region = "" - if err := c.Prepare(); len(err) != 1 { - t.Fatalf("err: %s", err) - } - - c.Region = "us-east-12" - if err := c.Prepare(); len(err) != 1 { - t.Fatalf("err: %s", err) - } - - c.Region = "us-east-1" - if err := c.Prepare(); len(err) != 0 { - t.Fatalf("err: %s", err) - } -} - func TestRunConfigPrepare_SourceAmi(t *testing.T) { c := testConfig() c.SourceAmi = "" diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 22018246b..d0767d4a8 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -62,12 +62,13 @@ func (s *StepRunSourceInstance) Run(state map[string]interface{}) multistep.Step ui.Say(fmt.Sprintf("Waiting for instance (%s) to become ready...", s.instance.InstanceId)) stateChange := StateChangeConf{ Conn: ec2conn, - Instance: s.instance, Pending: []string{"pending"}, Target: "running", + Refresh: InstanceStateRefreshFunc(ec2conn, s.instance), StepState: state, } - s.instance, err = WaitForState(&stateChange) + latestInstance, err := WaitForState(&stateChange) + s.instance = latestInstance.(*ec2.Instance) if err != nil { err := fmt.Errorf("Error waiting for instance (%s) to become ready: %s", s.instance.InstanceId, err) state["error"] = err @@ -95,10 +96,10 @@ func (s *StepRunSourceInstance) Cleanup(state map[string]interface{}) { } stateChange := StateChangeConf{ - Conn: ec2conn, - Instance: s.instance, - Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"}, - Target: "running", + Conn: ec2conn, + Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"}, + Refresh: InstanceStateRefreshFunc(ec2conn, s.instance), + Target: "running", } WaitForState(&stateChange) diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 46f5231fc..714c96afc 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -8,7 +8,6 @@ package ebs import ( "errors" "fmt" - "github.com/mitchellh/goamz/aws" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" awscommon "github.com/mitchellh/packer/builder/amazon/common" @@ -67,9 +66,9 @@ func (b *Builder) Prepare(raws ...interface{}) error { } func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { - region, ok := aws.Regions[b.config.Region] - if !ok { - panic("region not found") + region, err := b.config.Region() + if err != nil { + return nil, err } auth, err := b.config.AccessConfig.Auth() diff --git a/builder/amazon/ebs/step_create_ami.go b/builder/amazon/ebs/step_create_ami.go index 0b08bc7c9..5f75fb423 100644 --- a/builder/amazon/ebs/step_create_ami.go +++ b/builder/amazon/ebs/step_create_ami.go @@ -52,7 +52,7 @@ func (s *stepCreateAMI) Run(state map[string]interface{}) multistep.StepAction { // Set the AMI ID in the state ui.Say(fmt.Sprintf("AMI: %s", createResp.ImageId)) amis := make(map[string]string) - amis[config.Region] = createResp.ImageId + amis[ec2conn.Region.Name] = createResp.ImageId state["amis"] = amis // Wait for the image to become ready diff --git a/builder/amazon/ebs/step_stop_instance.go b/builder/amazon/ebs/step_stop_instance.go index f0c7b47a9..6f6add6c8 100644 --- a/builder/amazon/ebs/step_stop_instance.go +++ b/builder/amazon/ebs/step_stop_instance.go @@ -29,12 +29,13 @@ func (s *stepStopInstance) Run(state map[string]interface{}) multistep.StepActio ui.Say("Waiting for the instance to stop...") stateChange := awscommon.StateChangeConf{ Conn: ec2conn, - Instance: instance, Pending: []string{"running", "stopping"}, Target: "stopped", + Refresh: awscommon.InstanceStateRefreshFunc(ec2conn, instance), StepState: state, } - instance, err = awscommon.WaitForState(&stateChange) + instanceRaw, err := awscommon.WaitForState(&stateChange) + instance = instanceRaw.(*ec2.Instance) if err != nil { err := fmt.Errorf("Error waiting for instance to stop: %s", err) state["error"] = err diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 83e486a49..ffbdfb467 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -5,7 +5,6 @@ package instance import ( "errors" "fmt" - "github.com/mitchellh/goamz/aws" "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" awscommon "github.com/mitchellh/packer/builder/amazon/common" @@ -134,9 +133,9 @@ func (b *Builder) Prepare(raws ...interface{}) error { } func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) { - region, ok := aws.Regions[b.config.Region] - if !ok { - panic("region not found") + region, err := b.config.Region() + if err != nil { + return nil, err } auth, err := b.config.AccessConfig.Auth() diff --git a/builder/amazon/instance/step_register_ami.go b/builder/amazon/instance/step_register_ami.go index d95deb888..1accf7e48 100644 --- a/builder/amazon/instance/step_register_ami.go +++ b/builder/amazon/instance/step_register_ami.go @@ -50,7 +50,7 @@ func (s *StepRegisterAMI) Run(state map[string]interface{}) multistep.StepAction // Set the AMI ID in the state ui.Say(fmt.Sprintf("AMI: %s", registerResp.ImageId)) amis := make(map[string]string) - amis[config.Region] = registerResp.ImageId + amis[ec2conn.Region.Name] = registerResp.ImageId state["amis"] = amis // Wait for the image to become ready diff --git a/communicator/ssh/communicator.go b/communicator/ssh/communicator.go index 3af09e1d0..6f729e9f6 100644 --- a/communicator/ssh/communicator.go +++ b/communicator/ssh/communicator.go @@ -87,6 +87,7 @@ func (c *comm) Start(cmd *packer.RemoteCmd) (err error) { } } + log.Printf("remote command exited with '%d': %s", exitStatus, cmd.Command) cmd.SetExited(exitStatus) }() diff --git a/config.go b/config.go index 4660fad1c..e2b701374 100644 --- a/config.go +++ b/config.go @@ -20,6 +20,7 @@ const defaultConfig = ` "builders": { "amazon-ebs": "packer-builder-amazon-ebs", + "amazon-chroot": "packer-builder-amazon-chroot", "amazon-instance": "packer-builder-amazon-instance", "digitalocean": "packer-builder-digitalocean", "virtualbox": "packer-builder-virtualbox", diff --git a/plugin/builder-amazon-chroot/main.go b/plugin/builder-amazon-chroot/main.go new file mode 100644 index 000000000..b7d71df44 --- /dev/null +++ b/plugin/builder-amazon-chroot/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/mitchellh/packer/builder/amazon/chroot" + "github.com/mitchellh/packer/packer/plugin" +) + +func main() { + plugin.ServeBuilder(new(chroot.Builder)) +} diff --git a/plugin/builder-amazon-chroot/main_test.go b/plugin/builder-amazon-chroot/main_test.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/plugin/builder-amazon-chroot/main_test.go @@ -0,0 +1 @@ +package main diff --git a/website/source/docs/builders/amazon-chroot.html.markdown b/website/source/docs/builders/amazon-chroot.html.markdown new file mode 100644 index 000000000..4839344e4 --- /dev/null +++ b/website/source/docs/builders/amazon-chroot.html.markdown @@ -0,0 +1,177 @@ +--- +layout: "docs" +page_title: "Amazon AMI Builder (chroot)" +--- + +# AMI Builder (chroot) + +Type: `amazon-chroot` + +The `amazon-chroot` builder is able to create Amazon AMIs backed by +an EBS volume as the root device. For more information on the difference +between instance storage and EBS-backed instances, see the +["storage for the root device" section in the EC2 documentation](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ComponentsAMIs.html#storage-for-the-root-device). + +The difference between this builder and the `amazon-ebs` builder is that +this builder is able to build an EBS-backed AMI without launching a new +EC2 instance. This can dramatically speed up AMI builds for organizations +who need the extra fast build. + +
+

This is an advanced builder. If you're just getting +started with Packer, we recommend starting with the +amazon-ebs builder, which is +much easier to use.

+
+ +The builder does _not_ manage AMIs. Once it creates an AMI and stores it +in your account, it is up to you to use, delete, etc. the AMI. + +## How Does it Work? + +This builder works by creating a new EBS volume from an existing source AMI +and attaching it into an already-running EC2 instance. One attached, a +[chroot](http://en.wikipedia.org/wiki/Chroot) is used to provision the +system within that volume. After provisioning, the volume is detached, +snapshotted, and an AMI is made. + +Using this process, minutes can be shaved off the AMI creation process +because a new EC2 instance doesn't need to be launched. + +There are some restrictions, however. The host EC2 instance where the +volume is attached to must be a similar system (generally the same OS +version, kernel versions, etc.) as the AMI being built. Additionally, +this process is much more expensive because the EC2 instance must be kept +running persistently in order to build AMIs, whereas the other AMI builders +start instances on-demand to build AMIs as needed. + +## Configuration Reference + +There are many configuration options available for the builder. They are +segmented below into two categories: required and optional parameters. Within +each category, the available configuration keys are alphabetized. + +Required: + +* `access_key` (string) - The access key used to communicate with AWS. + If not specified, Packer will attempt to read this from environmental + variables `AWS_ACCESS_KEY_ID` or `AWS_ACCESS_KEY` (in that order). + If the environmental variables aren't set and Packer is running on + an EC2 instance, Packer will check the instance metadata for IAM role + keys. + +* `ami_name` (string) - The name of the resulting AMI that will appear + when managing AMIs in the AWS console or via APIs. This must be unique. + To help make this unique, certain template parameters are available for + this value, which are documented below. + +* `secret_key` (string) - The secret key used to communicate with AWS. + If not specified, Packer will attempt to read this from environmental + variables `AWS_SECRET_ACCESS_KEY` or `AWS_SECRET_KEY` (in that order). + If the environmental variables aren't set and Packer is running on + an EC2 instance, Packer will check the instance metadata for IAM role + keys. + +* `source_ami` (string) - The source AMI whose root volume will be copied + and provisioned on the currently running instance. This must be an + EBS-backed AMI with a root volume snapshot that you have access to. + +Optional: + +* `chroot_mounts` (list of list of strings) - This is a list of additional + devices to mount into the chroot environment. This configuration parameter + requires some additional documentation which is in the "Chroot Mounts" section + below. Please read that section for more information on how to use this. + +* `copy_files` (list of strings) - Paths to files on the running EC2 instance + that will be copied into the chroot environment prior to provisioning. + This is useful, for example, to copy `/etc/resolv.conf` so that DNS lookups + work. + +* `device_path` (string) - The path to the device where the root volume + of the source AMI will be attached. This defaults to "" (empty string), + which forces Packer to find an open device automatically. + +* `mount_command` (string) - The command to use to mount devices. This + defaults to "mount". This may be useful to set if you want to set + environmental variables or perhaps run it with `sudo` or so on. + +* `mount_path` (string) - The path where the volume will be mounted. This is + where the chroot environment will be. This defaults to + `packer-amazon-chroot-volumes/{{.Device}}`. This is a configuration + template where the `.Device` variable is replaced with the name of the + device where the volume is attached. + +* `unmount_command` (string) - Just like `mount_command`, except this is + the command to unmount devices. + +## Basic Example + +Here is a basic example. It is completely valid except for the access keys: + +
+{
+  "type": "amazon-chroot",
+  "access_key": "YOUR KEY HERE",
+  "secret_key": "YOUR SECRET KEY HERE",
+  "source_ami": "ami-e81d5881",
+  "ami_name": "packer-amazon-chroot {{.CreateTime}}"
+}
+
+ +## AMI Name Variables + +The AMI name specified by the `ami_name` configuration variable is actually +treated as a [configuration template](/docs/templates/configuration-templates.html). +Packer provides a set of variables that it will replace +within the AMI name. This helps ensure the AMI name is unique, as AWS requires. + +The available variables are shown below: + +* `CreateTime` - This will be replaced with the Unix timestamp of when + the AMI was built. + +## Chroot Mounts + +The `chroot_mounts` configuration can be used to mount additional devices +within the chroot. By default, the following additional mounts are added +into the chroot by Packer: + +* `/proc` (proc) +* `/sys` (sysfs) +* `/dev` (bind to real `/dev`) +* `/dev/pts` (devpts) +* `/proc/sys/fs/binfmt_misc` (binfmt_misc) + +These default mounts are usually good enough for anyone and are sane +defaults. However, if you want to change or add the mount points, you may +using the `chroot_mounts` configuration. Here is an example configuration: + +
+{
+  "chroot_mounts": [
+    ["proc", "proc", "/proc"],
+    ["bind", "/dev", "/dev"]
+  ]
+}
+
+ +`chroot_mounts` is a list of a 3-tuples of strings. The three components +of the 3-tuple, in order, are: + +* The filesystem type. If this is "bind", then Packer will properly bind + the filesystem to another mount point. + +* The source device. + +* The mount directory. + +## Parallelism + +A quick note on parallelism: it is perfectly safe to run multiple +_separate_ Packer processes with the `amazon-chroot` builder on the same +EC2 instance. In fact, this is recommended as a way to push the most performance +out of your AMI builds. + +Packer properly obtains a process lock for the parallelism-sensitive parts +of its internals such as finding an available device. diff --git a/website/source/docs/builders/amazon.html.markdown b/website/source/docs/builders/amazon.html.markdown index b9dc7d2e7..604f2489a 100644 --- a/website/source/docs/builders/amazon.html.markdown +++ b/website/source/docs/builders/amazon.html.markdown @@ -18,6 +18,13 @@ AMI. Packer supports the following builders at the moment: instance-store AMIs by launching and provisioning a source instance, then rebundling it and uploading it to S3. +* [amazon-chroot](/docs/builders/amazon-chroot.html) - Create EBS-backed AMIs + from an existing EC2 instance by mounting the root device and using a + [Chroot](http://en.wikipedia.org/wiki/Chroot) environment to provision + that device. This is an **advanced builder and should not be used by + newcomers**. However, it is also the fastest way to build an EBS-backed + AMI since no new EC2 instance needs to be launched. +
Don't know which builder to use? If in doubt, use the amazon-ebs builder. It is