Preserves file system metadata between new build and latest shipping image.

This script preserves the root file system metadata as much as possible
between the specified image and the latest shipping image. It preserves
the metadata by ensuring that the files reuse the same inodes and that they
are located at the same physical location on-disk. This leads to smaller
auto-update delta payload and less disk reshuffling, extending the life
of the SSD.

It is called before the image is signed during the stamping process.

Currently, this only supports x86-mario.

This is a continuation of a previous CL located at:
http://codereview.chromium.org/6058006/

BUG=chromium-os:10188
TEST=Build image, boot image, auto-update to new image, run suite_Smoke

Change-Id: I3270245dc15a074abb3bac250922c30e2e105f92

Review URL: http://codereview.chromium.org/6079004

2 files changed, 309 insertions, 1 deletions

diff --git a/scripts/image_signing/align_rootfs.sh b/scripts/image_signing/align_rootfs.sh
new file mode 100755
index 00000000..5af62020
--- /dev/null
+++ b/scripts/image_signing/align_rootfs.sh
@@ -0,0 +1,237 @@
+#!/bin/bash
+
+# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Script to preserve the on-disk file layout of the specified image and
+# the latest shipping image.  This is accomplished by copying the new rootfs
+# over a template rootfs (aka the latest shipping image) to preserve as much
+# of the metadata from the shipping rootfs as possible. This will ensure
+# minimal disk shuffling when applying the auto-update.
+#
+# Note: This script does not recompute the rootfs hash.
+
+# Load common library.  This should be the first executable line.
+# The path to common.sh should be relative to your script's location.
+. "$(dirname "$0")/common.sh"
+
+load_shflags
+
+# Flags.
+DEFINE_string app_id "{87efface-864d-49a5-9bb3-4b050a7c227a}" \
+  "App ID to use when pinging the Omaha server for the latest version."
+DEFINE_string channel "dev-channel" \
+  "Release channel of the build."
+DEFINE_string hardware_id "IEC MARIO PONY 6101" \
+  "The Hardware ID of the latest shipping image to use as a template."
+DEFINE_string image "" \
+  "The image that needs to be aligned to the latest shipping image."
+
+IMAGE_SEARCH_STRING=
+RELEASE_URL=
+RELEASE_ALT_URL=
+
+# Sets up environment variables specific to the board.
+initialize() {
+  if [ "${FLAGS_hardware_id}" = "IEC MARIO PONY 6101" ]; then
+    IMAGE_SEARCH_STRING="*SSD_MP_SIGNED.bin"
+    RELEASE_URL="http://chromeos-images/chromeos-official/${FLAGS_channel}/x86-mario"
+    RELEASE_ALT_URL="${RELEASE_URL}-rc"
+  else
+    die "Hardware ID \"${FLAGS_hardware_id}\" not supported"
+  fi
+}
+
+# Gets the latest shipping version for the specified Hardware ID by pinging
+# the Omaha server.  The latest shipping version is printed to stdout.
+# The caller must use process substition when invoking this function so we
+# use a simple trap for clean up instead of calling add_cleanup_action.
+get_latest_shipping_version() {
+  info "Pinging Omaha for the latest shipping version."
+  local auserver_url="https://tools.google.com/service/update2"
+  local au_request_file=$(mktemp "/tmp/align_rootfs_au_request.XXXX")
+  trap "rm -f \"${au_request_file}\"" INT TERM EXIT
+  cat > "${au_request_file}" << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<o:gupdate xmlns:o="http://www.google.com/update2/request"
+   version="ChromeOSUpdateEngine-0.1.0.0"
+   updaterversion="ChromeOSUpdateEngine-0.1.0.0" protocol="2.0" ismachine="1">
+    <o:os version="Indy" platform="Chrome OS" sp="ForcedUpdate_i686"></o:os>
+    <o:app appid="${FLAGS_app_id}"
+       version="0.0.0.0" lang="en-US" track="${FLAGS_channel}"
+       hardware_class="${FLAGS_hardware_id}" delta_okay="true">
+        <o:updatecheck></o:updatecheck>
+    </o:app>
+</o:gupdate>
+EOF
+
+  wget -q --header="Content-Type: text/xml" \
+    --post-file="${au_request_file}" -O - ${auserver_url} |
+    sed 's/.*\(ChromeOSVersion="\)\([0-9\.]*\)"\(.*\)/\2/'
+}
+
+# Downloads the image from the specified |URL| using ${IMAGE_SEARCH_STRING} as
+# a wildcard match and write the image to |OUTPUT_FILE|.
+# Args: URL OUTPUT_FILE
+download_image() {
+  # Since we don't know the exact name, we'll just download recursively based
+  # on a wildcard. Then we'll rename that download file to the desired output
+  # file. It's important that the IMAGE_SEARCH_STRING matches only one file.
+  local url=$1
+  local output_file=$2
+  local retcode=0
+
+  local download_dir=$(mktemp -d "/tmp/align_rootfs_download_dir.XXXX")
+  add_cleanup_action "sudo rm -rf \"${download_dir}\""
+
+  wget -A "${IMAGE_SEARCH_STRING}" --progress=bar -r -l1 -nd \
+    -P "${download_dir}" ${url} || retcode=$?
+  if [ ${retcode} -eq 0 ]; then
+    mv -f "${download_dir}"/* "${output_file}"
+  else
+    return ${retcode}
+  fi
+}
+
+# Retrieves the latest shipping image and saves it to |IMAGE|.
+# Args: VERSION IMAGE
+get_shipping_image() {
+  # The image may reside in one of two URLs. We don't know exactly which one
+  # so we'll have to try both.
+  local version=$1
+  local image=$2
+  local url="${RELEASE_URL}/${version}"
+  download_image "${url}" "${image}" || retcode=$?
+  if [ ${retcode} -gt 0 ]; then
+    url="${RELEASE_ALT_URL}/${version}"
+    download_image "${url}" "${image}"
+  fi
+}
+
+# Copies the rootfs from |SRC_IMAGE| to the |DST_ROOT_FS| and preserves as
+# much of the file system metadata in |DST_ROOT_FS| as possible.
+# Args: SRC_IMAGE DST_ROOT_FS
+copy_root_fs() {
+  local src_image=$1
+  local dst_root_fs=$2
+
+  # Mount the src and dst rootfs.
+  local src_root_fs_dir=$(mktemp -d "/tmp/align_root_fs_src_mount_dir.XXXX")
+  add_cleanup_action "sudo rm -rf \"${src_root_fs_dir}\""
+  mount_image_partition_ro "${src_image}" 3 "${src_root_fs_dir}"
+  add_cleanup_action "sudo umount -d \"${src_root_fs_dir}\""
+
+  local dst_root_fs_dir=$(mktemp -d "/tmp/align_root_fs_dst_mount_dir.XXXX")
+  add_cleanup_action "sudo rm -rf \"${dst_root_fs_dir}\""
+  sudo mount -o loop "${dst_root_fs}" "${dst_root_fs_dir}" -o loop
+  add_cleanup_action "sudo umount -d \"${dst_root_fs_dir}\""
+
+  # Temporarily make immutable files on the dst rootfs mutable.
+  # We'll need to track these files in ${immutable_files} so we can make them
+  # mutable again.
+  local immutable_files=()
+  sudo find "${dst_root_fs_dir}" -xdev -type f |
+    while read -r file; do
+      immutable=$(sudo lsattr "${file}" | cut -d' ' -f1 | grep -q i ; echo $?)
+      if [ $immutable -eq 0 ]; then
+        immutable_files=("${immutable_files[@]}" "${file}")
+        sudo chattr -i "${file}"
+      fi
+    done
+
+  # Copy files from the src rootfs over top of dst rootfs.
+  # Use the --inplace flag to preserve as much of the file system metadata
+  # as possible.
+  sudo rsync -v -a -H -A -x --force --inplace --numeric-ids --delete \
+      "${src_root_fs_dir}"/ "${dst_root_fs_dir}"
+
+  # Make immutable files immutable again.
+  for file in ${immutable_files[*]} ; do
+    sudo chattr +i "${file}"
+  done
+
+  # Unmount the src and dst root fs so that we can replace the rootfs later.
+  perform_latest_cleanup_action
+  perform_latest_cleanup_action
+  perform_latest_cleanup_action
+  perform_latest_cleanup_action
+}
+
+# Zeroes the rootfs free space in the specified image.
+# Args: IMAGE
+zero_root_fs_free_space() {
+  local image=$1
+  local root_fs_dir=$(mktemp -d "/tmp/align_rootfs_zero_free_space_dir.XXXX")
+  add_cleanup_action "sudo rm -rf \"${root_fs_dir}\""
+  mount_image_partition "${image}" 3 "${root_fs_dir}"
+  add_cleanup_action "sudo umount -d \"${root_fs_dir}\""
+
+  info "Zeroing free space in rootfs"
+  sudo dd if=/dev/zero of="${root_fs_dir}/filler" oflag=sync bs=4096 || true
+  sudo rm -f "${root_fs_dir}/filler"
+  sudo sync
+
+  perform_latest_cleanup_action
+  perform_latest_cleanup_action
+}
+
+main() {
+  # Parse command line.
+  FLAGS "$@" || exit 1
+  eval set -- "${FLAGS_ARGV}"
+
+  # Only now can we die on error.  shflags functions leak non-zero error codes,
+  # so will die prematurely if 'set -e' is specified before now.
+  set -e
+
+  # Make sure we have the required parameters.
+  if [ -z "${FLAGS_image}" ];  then
+    die "--image is required."
+  fi
+
+  if [ ! -f "${FLAGS_image}" ]; then
+    die "Cannot find the specified image."
+  fi
+
+  # Initialize some variables specific to the board.
+  initialize
+
+  # Download the latest shipping image.
+  local latest_shipping_version=$(get_latest_shipping_version)
+  local latest_shipping_image=$(mktemp "/tmp/align_rootfs_shipping_image.XXXX")
+  add_cleanup_action "sudo rm -f \"${latest_shipping_image}\""
+  info "Downloading image (${latest_shipping_version})"
+  get_shipping_image "${latest_shipping_version}" "${latest_shipping_image}"
+
+  # Make sure the two rootfs are the same size.
+  # If they are not, then there is nothing for us to do.
+  # Note: Exit with a zero code so we do not break the build workflow.
+  local shipping_root_fs_size=$(partsize "${latest_shipping_image}" 3)
+  local new_root_fs_size=$(partsize "${FLAGS_image}" 3)
+  if [ ${shipping_root_fs_size} -ne ${new_root_fs_size} ]; then
+    warn "The latest shipping rootfs and the new rootfs are not the same size."
+    exit 0
+  fi
+
+  # Extract the rootfs from the shipping image and use this as a template
+  # for the new image.
+  temp_root_fs=$(mktemp "/tmp/align_rootfs_temp_rootfs.XXXX")
+  add_cleanup_action "sudo rm -f \"${temp_root_fs}\""
+  info "Extracting rootfs from shipping image"
+  extract_image_partition "${latest_shipping_image}" 3 "${temp_root_fs}"
+  enable_rw_mount "${temp_root_fs}"
+
+  # Perform actual copy of the two root file systems.
+  info "Copying rootfs"
+  copy_root_fs "${FLAGS_image}" "${temp_root_fs}"
+
+  # Replace the rootfs in the new image with the aligned version.
+  info "Replacing rootfs"
+  replace_image_partition "${FLAGS_image}" 3 "${temp_root_fs}"
+
+  # Zero rootfs free space.
+  zero_root_fs_free_space "${FLAGS_image}"
+}
+
+main "$@"
diff --git a/scripts/image_signing/common.sh b/scripts/image_signing/common.sh
index 5a3e600b..5465feb8 100755
--- a/scripts/image_signing/common.sh
+++ b/scripts/image_signing/common.sh
@@ -12,6 +12,75 @@ GPT=cgpt
 # The tag when the rootfs is changed.
 TAG_NEEDS_TO_BE_SIGNED="/root/.need_to_be_signed"
 
+# Array of actions that are executed during the clean up process.
+declare -a cleanup_actions
+
+# Adds an action to be executed during the clean up process.
+# Actions are executed in the reverse order of when they were added.
+# ARGS: ACTION
+add_cleanup_action() {
+  cleanup_actions[${#cleanup_actions[*]}]=$1
+}
+
+# Performs the latest clean up action and removes it from the list.
+perform_latest_cleanup_action() {
+  local num_actions=${#cleanup_actions[*]}
+  if [ ${num_actions} -gt 0 ]; then
+    eval "${cleanup_actions[$num_actions-1]}" > /dev/null 2>&1
+    unset cleanup_actions[$num_actions-1]
+  fi
+}
+
+# Performs clean up by executing actions in the cleanup_actions array in
+# reversed order.
+cleanup() {
+  set +e
+
+  while [ ${#cleanup_actions[*]} -gt 0 ]; do
+    perform_latest_cleanup_action
+  done
+
+  set -e
+}
+
+# ANSI color codes used when displaying messages.
+# Taken from src/scripts/common.sh.
+V_RED="\e[31m"
+V_YELLOW="\e[33m"
+V_BOLD_GREEN="\e[1;32m"
+V_BOLD_RED="\e[1;31m"
+V_BOLD_YELLOW="\e[1;33m"
+V_VIDOFF="\e[0m"
+
+# Prints an informational message.
+# Taken from src/scripts/common.sh.
+# Arg: MESSAGE
+info() {
+  echo -e >&2 "${V_BOLD_GREEN}INFO   : $1${V_VIDOFF}"
+}
+
+# Prints a warning message.
+# Taken from src/scripts/common.sh.
+# Arg: MESSAGE
+warn() {
+  echo -e >&2 "${V_BOLD_YELLOW}WARNING: $1${V_VIDOFF}"
+}
+
+# Prints the specified error and exit the script with an error code.
+# Taken from src/scripts/common.sh.
+# Args: MESSAGE
+error() {
+  echo -e >&2   "${V_BOLD_RED}ERROR  : $1${V_VIDOFF}"
+}
+
+# Prints an error message and exit with an error code.
+# Taken from src/scripts/common.sh.
+# Args: MESSAGE
+die() {
+  error "$1"
+  exit 1
+}
+
 # Finds and loads the 'shflags' library, or return as failed.
 load_shflags() {
   # Load shflags
@@ -245,5 +314,7 @@ no_chronos_password() {
   sudo grep -q '^chronos:\*:' "$rootfs/etc/shadow"
 }
 
-trap "cleanup_temps_and_mounts" EXIT
+trap "cleanup" INT TERM EXIT
+
+add_cleanup_action "cleanup_temps_and_mounts"