#!/bin/bash -eu

################### TO DO ###################
#
# * Check hg is installed
# * Check hg clone is working (error code 255)
# * Check virtualenv is installed
# * Add an option not to refresh tools repo
# * Check connectivity to an arbitrary master and report nicely
# * Summarize failures from manage_masters.py at end of log
# * Option to checkconfig only without update
# * Only update wiki if there really are changes
# * Only reconfig if there really are changes, or if forced
# * Send console output also to a log file
#
#############################################

# Include our shared function library
if [ -e ../../lib/shell/functions ]; then
    . ../../lib/shell/functions
else
    echo "Shared functions missing: ../../lib/shell/functions" >&2
    exit 1
fi

START_TIME="$(date +%s)"

# Explicitly unset any pre-existing environment variables to avoid variable collision
unset PREPARE_ONLY FORCE_RECONFIG MERGE_TO_PRODUCTION UPDATE_BUGZILLA UPDATE_WIKI RECONFIG_DIR RECONFIG_CREDENTIALS_FILE WIKI_USERNAME WIKI_PASSWORD

# Force python into unbuffered mode, so we get full output, as it happens from the fabric scripts
export PYTHONUNBUFFERED=1

function usage {
    echo
    echo "This script can be used to reconfig interactively, or non-interactively. It will merge"
    echo "buildbotcustom, buildbot-configs, mozharness from default to production(-0.8)."
    echo "It will then reconfig the buildbot masters, update the foopies with the latest tools"
    echo "changes, and afterwards if all was successful, it will also update the wiki page"
    echo "https://wiki.mozilla.org/ReleaseEngineering/Maintenance and additionally post comments"
    echo "on the Bugzilla bugs it finds in the commit messages of merged changes, providing a link"
    echo "to the hg web interface of the commits that have been deployed to production."
    echo
    echo "USAGE"
    echo "     1) $(basename "${0}") -h"
    echo "     2) $(basename "${0}") [-b] [-f] [-m] [-n] [-p] [-r RECONFIG_DIR] [-t] [-w RECONFIG_CREDENTIALS_FILE]"
    echo
    echo "OPTIONS"
    echo "    -b:                            No posting of comments to Bugzilla."
    echo "    -f:                            Force reconfig, even if no changes merged."
    echo "    -h:                            Display help."
    echo "    -m:                            No merging of default -> production(-0.8) of hg branches."
    echo "    -n:                            No wiki update."
    echo "    -p:                            Prepare only; does not push changes to hg, nor perform"
    echo "                                   reconfig, nor update wiki, nor post to Bugzilla. Useful"
    echo "                                   for validating setup, or resolving merge conflicts in"
    echo "                                   advance early rather than waiting until real reconfig to"
    echo "                                   resolve conflicts."
    echo "    -r RECONFIG_DIR:               Use directory RECONFIG_DIR for storing temporary files"
    echo "                                   (default is /tmp/reconfig). This directory, and any"
    echo "                                   necessary parent directories will be created if required."
    echo "    -w RECONFIG_CREDENTIALS_FILE:  Source environment variables: BUGZILLA_USERNAME,"
    echo "                                   BUGZILLA_PASSWORD, WIKI_USERNAME, WIKI_PASSWORD from file"
    echo "                                   RECONFIG_CREDENTIALS_FILE (default is ~/.reconfig/config)."
    echo
    echo "EXIT CODES"
    echo "     0        Success"
    echo "     1        Error loading shared functions"
    echo "     2        Bad command line options specified"
    echo "    64        Could not create directory to store results (RECONFIG_DIR)"
    echo "    65        Wiki credentials file not found"
    echo "    66        Python 2.7 not found in PATH"
    echo "    67        Reconfig aborted by user after incomplete reconfig found"
    echo "    68        Aborted due to incomplete reconfig found but non-interactive session, so"
    echo "              could not ask user how to proceed"
    echo "    69        Error during hg merge (most likely due to merge conflict(s))"
    echo "    70        Theoretically not possible - update_maintenance_wiki.sh tries and fails"
    echo "              to create the RECONFIG_DIR when it detects it does not exist (but should"
    echo "              have already been created by end_to_end_reconfig.sh)"
    echo "    71        Theoretically not possible - no wiki markdown file passed to"
    echo "              update_maintenance_wiki.sh"
    echo "    72        Theoretically not possible - non-existing markdown file passed to"
    echo "              update_maintenance_wiki.sh"
    echo "    73        WIKI_USERNAME not specified in reconfig credentials file"
    echo "    74        WIKI_PASSWORD not specified in reconfig credentials file"
    echo "    75        Could not retreieve login token from wiki - probably a connectivity issue"
    echo "    76        Wiki user/password not authorized to update wiki page"
    echo "              https://wiki.mozilla.org/ReleaseEngineering/Maintenance"
    echo "    77        Wiki API provided a reason not to supply an edit token for the wiki page,"
    echo "              which is something other than user/password not authorized (which would"
    echo "              result in exit code 76)"
    echo "    78        Wiki API response to request to provide an edit token produced a"
    echo "              non-parsable response"
    echo "    79        BUGZILLA_USERNAME not specified in reconfig credentials file"
    echo "    80        BUGZILLA_PASSWORD not specified in reconfig credentials file"
    echo "    81        Bugzilla user/password not authorized to update Bugzilla"
    echo "    82        Could not install python packages into a virtualenv"
    echo
}

# Simple function to output the name of this script and the options that were passed to it
function command_called {
    echo -n "Command called:"
    for ((INDEX=0; INDEX<=$#; INDEX+=1))
    do
        echo -n " '${!INDEX}'"
    done
    echo ''
    echo "From directory: '$(pwd)'"
}

# writes hg commands to the hg log, and just summarizes command called in main log
function hg_wrapper {
    if [ $1 == 'clone' ]; then
        command=('hg' "${@}" "${RECONFIG_DIR}/${repo}")
    else
        command=('hg' -R "${RECONFIG_DIR}/${repo}" "${@}")
    fi
    command_full="Running:"
    for ((INDEX=0; INDEX<${#command[@]}; INDEX+=1)); do
        command_full="${command_full} '${command[$INDEX]}'"
    done
    echo "  * ${command_full}"
    {
        echo
        echo "$(date): ${command_full}"
        echo
        HG_START="$(date +%s)"
        set +e
        "${command[@]}"
        HG_RETURN_CODE=$?
        set -e
        HG_STOP="$(date +%s)"
        echo
        echo "$(date): Completed ($((HG_STOP - HG_START))s)"
    } >>"${RECONFIG_DIR}/hg-${START_TIME}.log" 2>&1
    return "${HG_RETURN_CODE}"
}

set +u
command_called "${@}" | sed '1s/^/  * /;2s/^/    /'
set -u

echo "  * Start timestamp: ${START_TIME}"
echo "  * Parsing parameters of $(basename "${0}")..."
# Parse parameters passed to this script
while getopts ":bfhmnpr:w:" opt; do
    case "${opt}" in
        b)  UPDATE_BUGZILLA=0
            ;;
        f)  FORCE_RECONFIG=1
            ;;
        h)  echo "  * Help option requested"
            usage
            exit 0
            ;;
        m)  MERGE_TO_PRODUCTION=0
            ;;
        n)  UPDATE_WIKI=0
            ;;
        p)  PREPARE_ONLY=1
            ;;
        r)  RECONFIG_DIR="${OPTARG}"
            ;;
        w)  RECONFIG_CREDENTIALS_FILE="${OPTARG}"
            ;;
        ?)  usage >&2
            exit 2
            ;;
    esac
done

echo "  * Setting defaults for parameters not provided in command line options..."

PREPARE_ONLY="${PREPARE_ONLY:-0}"
FORCE_RECONFIG="${FORCE_RECONFIG:-0}"
MERGE_TO_PRODUCTION="${MERGE_TO_PRODUCTION:-1}"
UPDATE_BUGZILLA="${UPDATE_BUGZILLA:-1}"
UPDATE_WIKI="${UPDATE_WIKI:-1}"
RECONFIG_DIR="${RECONFIG_DIR:-/tmp/reconfig}"
RECONFIG_CREDENTIALS_FILE="${RECONFIG_CREDENTIALS_FILE:-${HOME}/.reconfig/config}"

RECONFIG_UPDATE_FILE="reconfig_update_for_maintenance.wiki"

##### Now check parsed parameters are valid...

echo "  * Validating parameters..."

if [ "${PREPARE_ONLY}" == 0 ]; then
    echo "  * Will be preparing *and performing* reconfig, all being well"
else
    echo "  * Preparing reconfig only; will not enact changes"
fi

if [ ! -d "${RECONFIG_DIR}" ]; then
    echo "  * Storing reconfig output under '${RECONFIG_DIR}'..."
    if ! mkdir -p "${RECONFIG_DIR}"; then
        echo "ERROR: Directory '${RECONFIG_DIR}' could not be created from directory '$(pwd)'" >&2
        exit 64
    fi
else
    echo "  * Reconfig directory '${RECONFIG_DIR}' exists - OK"
fi

# Convert ${RECONFIG_DIR} to an absolute directory, in case it is relative, by stepping into it...
pushd "${RECONFIG_DIR}" >/dev/null
if [ "${RECONFIG_DIR}" != "$(pwd)" ]; then
    echo "  * Reconfig directory absolute path: '$(pwd)'"
fi
RECONFIG_DIR="$(pwd)"
popd >/dev/null

# To avoid user getting confused about parent directory, tell user the
# absolute path of the credentials file...
echo "  * Checking we have a reconfig credentials file..."
RECONFIG_CREDS_PARENT_DIR="$(dirname "${RECONFIG_CREDENTIALS_FILE}")"
if [ ! -e "${RECONFIG_CREDS_PARENT_DIR}" ]; then
echo "  * Reconfig credentials file parent directory '${RECONFIG_CREDS_PARENT_DIR}' not found; creating..."
    mkdir -p "${RECONFIG_CREDS_PARENT_DIR}"
fi
pushd "${RECONFIG_CREDS_PARENT_DIR}" >/dev/null
ABS_RECONFIG_CREDENTIALS_FILE="$(pwd)/$(basename "${RECONFIG_CREDENTIALS_FILE}")"
popd >/dev/null
if [ "${RECONFIG_CREDENTIALS_FILE}" == "${ABS_RECONFIG_CREDENTIALS_FILE}" ]; then
    echo "  * Reconfig credentials file location: '${RECONFIG_CREDENTIALS_FILE}'"
else
    echo "  * Reconfig credentials file location: '${RECONFIG_CREDENTIALS_FILE}' (absolute path: '${ABS_RECONFIG_CREDENTIALS_FILE}')"
fi
if [ ! -e "${ABS_RECONFIG_CREDENTIALS_FILE}" ]; then
    echo "  * Reconfig credentials file '${ABS_RECONFIG_CREDENTIALS_FILE}' not found; creating..."
    {
        echo "# Needed if updating wiki - note the wiki does *not* use your LDAP credentials..."
        echo "export WIKI_USERNAME='naughtymonkey'"
        echo "export WIKI_PASSWORD='nobananas'"
        echo
        echo "# Needed if updating Bugzilla bugs to mark them as in production - *no* Persona integration - must be a native Bugzilla account..."
        echo "# Details for the 'Release Engineering SlaveAPI Service' <slaveapi@mozilla.releng.tld> Bugzilla user can be found in the RelEng"
        echo "# private repo, in file passwords/slaveapi-bugzilla.txt.gpg (needs decrypting with an approved gpg key)."
        echo "export BUGZILLA_USERNAME='naughtymonkey'"
        echo "export BUGZILLA_PASSWORD='nobananas'"
    } > "${RECONFIG_CREDENTIALS_FILE}"
    echo "  * Created credentials file '${ABS_RECONFIG_CREDENTIALS_FILE}'. Please edit this file, setting appropriate values, then rerun."
    exit 65
else
    echo "  * Loading config from '${ABS_RECONFIG_CREDENTIALS_FILE}'..."
    source "${RECONFIG_CREDENTIALS_FILE}"
fi

# Now step into directory this script is in...
cd "$(dirname "${0}")"

# Test python version, and availability of fabric...
echo "  * Checking python version is 2.7..."
if ! python --version 2>&1 | grep -q '^Python 2\.7'; then
    echo "ERROR: Python version 2.7 not found - please make sure python 2.7 is in your PATH" >&2
    exit 66
fi

for package in fabric requests; do
    installed_package=false
    echo "  * Checking ${package} package is available in python environment..."
    if ! python -c "import ${package}" >/dev/null 2>&1; then
        echo "  * Package ${package} not found"
        if [ ! -e "${RECONFIG_DIR}/reconfig-virtual-env" ]; then
            echo "  * Creating virtualenv directory '${RECONFIG_DIR}/reconfig-virtual-env' for reconfig tool..."
            echo "  * Logging to: '${RECONFIG_DIR}/virtualenv-installation.log'..."
            virtualenv "${RECONFIG_DIR}/reconfig-virtual-env" >"${RECONFIG_DIR}/virtualenv-installation.log" 2>&1
            set +u
            source "${RECONFIG_DIR}/reconfig-virtual-env/bin/activate"
            set -u
            echo "  * Installing ${package} under '${RECONFIG_DIR}/reconfig-virtual-env'..."
            pip install "${package}" >>"${RECONFIG_DIR}/virtualenv-installation.log" 2>&1
            installed_package=true
        else
            echo "  * Attempting to use existing virtualenv found in '${RECONFIG_DIR}/reconfig-virtual-env'..."
            set +u
            source "${RECONFIG_DIR}/reconfig-virtual-env/bin/activate"
            set -u
            if ! python -c "import ${package}" >/dev/null 2>&1; then
                echo "  * Package ${package} not found under virtualenv '${RECONFIG_DIR}/reconfig-virtual-env', installing..."
                pip install "${package}" >"${RECONFIG_DIR}/virtualenv-installation.log" 2>&1
                installed_package=true
            else
                echo "  * Using existing package ${package} found in '${RECONFIG_DIR}/reconfig-virtual-env'"
            fi
        fi
    else
        echo "  * Package ${package} found"
    fi
    if "${installed_package}"; then
        echo "  * Re-checking if package ${package} is now available in python environment..."
        if ! python -c "import ${package}" >/dev/null 2>&1; then
            echo "ERROR: Could not successfully install package ${package} into python virtualenv '${RECONFIG_DIR}/reconfig-virtual-env'" >&2
            echo "Exiting..." >&2
            exit 82
        else
            echo "  * Package ${package} installed successfully into python environment"
        fi
    fi
done

if [ "${UPDATE_BUGZILLA}" == '1' ]; then
    echo "  * Updates to Bugzilla enabled"
    echo "  * Testing existence of Bugzilla login credentials..."
    if [ -z "${BUGZILLA_USERNAME}" ]; then
        echo "ERROR: Environment variable BUGZILLA_USERNAME must be set for posting comments to https://bugzilla.mozilla.org/" >&2
        echo "Exiting..." >&2
        exit 79
    else
        echo "  * Environment variable BUGZILLA_USERNAME defined ('${BUGZILLA_USERNAME}')"
    fi
    if [ -z "${BUGZILLA_PASSWORD}" ]; then
        echo "ERROR: Environment variable BUGZILLA_PASSWORD must be set for posting comments to https://bugzilla.mozilla.org/" >&2
        echo "Exiting..." >&2
        exit 80
    else
        echo "  * Environment variable BUGZILLA_PASSWORD defined"
    fi
fi

if [ "${UPDATE_WIKI}" == '1' ]; then
    echo "  * Wiki update enabled"
    # Now validate wiki credentials by performing a dry run...
    echo "  * Testing login credentials for wiki..."
    ./update_maintenance_wiki.sh -d
else
    echo "  * Not updating wiki"
fi

if [ "${UPDATE_BUGZILLA}" == '1' ]; then
    echo "  * Testing Bugzilla login as user '${BUGZILLA_USERNAME}'..."
    BUGZILLA_LOGIN_RESPONSE="$(mktemp -t bugzilla_login.XXXXXXXXXX)"
    curl -s -G --data-urlencode "login=${BUGZILLA_USERNAME}" --data-urlencode "password=${BUGZILLA_PASSWORD}" 'https://bugzilla.mozilla.org/rest/login' > "${BUGZILLA_LOGIN_RESPONSE}"
    if grep -q '"token":' "${BUGZILLA_LOGIN_RESPONSE}"; then
        echo "  * Login to Bugzilla successful"
    else
        echo "ERROR: Login to Bugzilla failed - response given:" >&2
        cat "${BUGZILLA_LOGIN_RESPONSE}" | sed 's/^/    /' >&2
        rm "${BUGZILLA_LOGIN_RESPONSE}"
        echo >&2
        echo "Please check values of BUGZILLA_USERNAME and BUGZILLA_PASSWORD set in this file: '${RECONFIG_CREDENTIALS_FILE}'" >&2
        echo "Exiting..." >&2
        exit 81
    fi
    rm "${BUGZILLA_LOGIN_RESPONSE}"
else
    echo "  * Not updating Bugzilla"
fi

# Check if a previous reconfig did not complete
if [ -f "${RECONFIG_DIR}/pending_changes" ]; then
    echo "  * It looks like a previous reconfig did not complete"
    echo "  * Checking if 'standard in' is connected to a terminal..."
    if [ -t 0 ]; then
        # 'standard in' is connected to a terminal, can ask user a question!
        echo "  * Please select one of the following options:"
        echo "        1) Continue with existing reconfig (e.g. if you have resolved a merge conflict)"
        echo "        2) Delete saved state for existing reconfig, and start from fresh"
        echo "        3) Abort and exit reconfig process"
        choice=''
        while [ "${choice}" != 1 ] && [ "${choice}" != 2 ] && [ "${choice}" != 3 ]; do
            echo -n "    Your choice: "
            read choice
        done
        case "${choice}" in
            1) echo "  * Continuing with stalled reconfig..."
               ;;
            2) echo "  * Cleaning out previous reconfig from '${RECONFIG_DIR}'..."
               rm -rf "${RECONFIG_DIR}"/{buildbot-configs,buildbotcustom,mozharness,pending_changes,${RECONFIG_UPDATE_FILE}}
               ;;
            3) echo "  * Aborting reconfig..."
               exit 67
               ;;
        esac
    else
        # 'standard in' not connected to a terminal, assume no user connected...
        echo "  * Non-interactive shell detected, cannot ask whether to continue or not, therefore aborting..."
        exit 68
    fi
fi

### If we get this far, all our preflight checks have passed, so now on to business...
echo "  * All preflight checks passed in '$(basename "${0}")'"

if [ "${PREPARE_ONLY}" == '0' ]; then
    # Failing to update IRC is non-fatal.
    IRC_MSG="Merge to production has started for buildbot repos."
    if [ "${FORCE_RECONFIG}" == '1' ]; then
        IRC_MSG="Reconfig has started."
    fi
    ./update_irc.sh "${IRC_MSG}" || true
fi

# Merges mozharness, buildbot-configs from default -> production.
# Merges buildbostcustom from default -> production-0.8.
# Returns 0 if something got merged, otherwise returns 1.
function merge_to_production {
    [ "${MERGE_TO_PRODUCTION}" == 0 ] && return 0
    echo "  * hg log for this session: '${RECONFIG_DIR}/hg-${START_TIME}.log'"
    # Create an empty merge file to compare against so we don't claim merge success when nothing was merged.
    {
        echo "Merging from default"
        echo
    } > "${RECONFIG_DIR}/empty_merge"
    for repo in mozharness buildbot-configs buildbotcustom; do
        if [ -d "${RECONFIG_DIR}/${repo}" ]; then
            echo "  * Existing hg clone of ${repo} found: '${RECONFIG_DIR}/${repo}' - pulling for updates..."
            hg_wrapper pull
        else
            echo "  * Cloning ssh://hg.mozilla.org/build/${repo} into '${RECONFIG_DIR}/${repo}'..."
            hg_wrapper clone "ssh://hg.mozilla.org/build/${repo}"
        fi
        if [ "${repo}" == 'buildbotcustom' ]; then
            branch='production-0.8'
        else
            branch='production'
        fi
        hg_wrapper up -r "${branch}"
        echo "  * Finding ${repo} changesets that would get merged from default to ${branch}..."
        {
            echo "Merging from default"
            echo
            hg -R "${RECONFIG_DIR}/${repo}" merge -P default
        } >> "${RECONFIG_DIR}/${repo}_preview_changes.txt" 2>/dev/null
        # Merging can fail if there are no changes between default and "${branch}"
        set +e
        hg_wrapper merge default
        RETVAL="${?}"
        set -e
        if [ "${RETVAL}" == '255' ]; then
            echo "  * No new changes found in ${repo}"
        elif [ "${RETVAL}" != '0' ]; then
            echo "ERROR: An error occurred during hg merge (exit code was ${RETVAL}). Please resolve conflicts/issues in '${RECONFIG_DIR}/${repo}',"
            echo "       push to ${branch} branch, and run this script again." >&2
            exit 69
        else
            echo "  * ${repo} merge resulted in change"
        fi
        # Still commit and push, even if no new changes, since a merge conflict might have been resolved
        hg_wrapper commit -l "${RECONFIG_DIR}/${repo}_preview_changes.txt"
        if [ "${PREPARE_ONLY}" == '0' ]; then
            echo "  * Pushing '${RECONFIG_DIR}/${repo}' ${branch} branch to ssh://hg.mozilla.org/build/${repo}..."
            hg_wrapper push
        fi
        if ! diff "${RECONFIG_DIR}/empty_merge" "${RECONFIG_DIR}/${repo}_preview_changes.txt" > /dev/null; then
  	    echo "${repo}" >> "${RECONFIG_DIR}/pending_changes"
        fi
    done
    [ -f "${RECONFIG_DIR}/pending_changes" ] && return 0 || return 1
}

# Now we process the commit messages from all the changes we landed. This is handled by the python script
# process_commit_comments.py. We pass options to this script based on what steps are enabled (e.g. wiki
# updates, bugzilla updates). Create an array for this, and if it is empty at the end, we know we don't
# have to do anything (arrays are better than strings since they can contains spaces).
commit_processing_options=()

if [ "${UPDATE_WIKI}" == '1' ]; then
    commit_processing_options+=('--wiki-markup-file' "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}")
fi
if [ "${UPDATE_BUGZILLA}" == '1' ] && [ "${PREPARE_ONLY}" == '0' ]; then
    commit_processing_options+=('--update-bugzilla')
fi

# Display the commands in advance in case the reconfig fails part way through.
if [ "${#commit_processing_options[@]}" -gt 0 ]; then
    echo "  * Providing post-processing commands in case reconfig aborts."
    echo -n "  * Will run '$(pwd)/process_commit_comments.py' --logdir '${RECONFIG_DIR}'"
    for ((i=0; i<${#commit_processing_options[@]}; i+=1)); do
        echo -n " '${commit_processing_options[${i}]}'"
    done
    echo
    echo "  * Will run: './update_maintenance_wiki.sh -r \"${RECONFIG_DIR}\" -w \"${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}\"'"
fi

# Return code of merge_to_production is 0 if merge performed successfully and changes made
if merge_to_production; then
    echo "  * All changes merged to production."
fi

if [ "${FORCE_RECONFIG}" == '1' ]; then
    production_masters_url='http://hg.mozilla.org/build/tools/raw-file/tip/buildfarm/maintenance/production-masters.json'
    if [ "${PREPARE_ONLY}" != '0' ]; then
        echo "  * Preparing reconfig only; not running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests show_revisions update"
        echo "  * Preparing reconfig only; not running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j32 -R scheduler -R build -R try -R tests checkconfig reconfig"
        echo "  * Preparing reconfig only; not running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests show_revisions"
    else
        echo "  * Fabric log for buildbot masters: '${RECONFIG_DIR}/manage_masters-${START_TIME}.log'"
        # Split into two steps so -j option can be varied between them
        echo "  * Running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests show_revisions update"
        ./manage_masters.py -f "${production_masters_url}" -j16 -R scheduler -R build -R try -R tests show_revisions update >>"${RECONFIG_DIR}/manage_masters-${START_TIME}.log" 2>&1
        echo "  * Running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j32 -R scheduler -R build -R try -R tests checkconfig reconfig"
        ./manage_masters.py -f "${production_masters_url}" -j32 -R scheduler -R build -R try -R tests checkconfig reconfig >>"${RECONFIG_DIR}/manage_masters-${START_TIME}.log" 2>&1
        # delete this now, since changes have been deployed
        [ -f "${RECONFIG_DIR}/pending_changes" ] && mv "${RECONFIG_DIR}/pending_changes" "${RECONFIG_DIR}/pending_changes_${START_TIME}"
        echo "  * Running: '$(pwd)/manage_masters.py' -f '${production_masters_url}' -j16 -R scheduler -R build -R try -R tests show_revisions"
        ./manage_masters.py -f "${production_masters_url}" -j16 -R scheduler -R build -R try -R tests show_revisions >>"${RECONFIG_DIR}/manage_masters-${START_TIME}.log" 2>&1
        echo "  * Reconfig of masters completed"
    fi
fi

if [ "${#commit_processing_options[@]}" -gt 0 ]; then
    echo -n "  * Running '$(pwd)/process_commit_comments.py' --logdir '${RECONFIG_DIR}'"
    for ((i=0; i<${#commit_processing_options[@]}; i+=1)); do echo -n " '${commit_processing_options[${i}]}'"; done
    echo
    ./process_commit_comments.py --logdir "${RECONFIG_DIR}" "${commit_processing_options[@]}"
else
    echo "  * Skipping commit messages processing step, since no wiki update nor Bugzilla update enabled, so not required"
fi

if [ -f "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}" ]; then
    if [ "${UPDATE_WIKI}" == "1" ]; then
        if [ "${PREPARE_ONLY}" != '0' ]; then
            echo "  * Running: './update_maintenance_wiki.sh -d -r \"${RECONFIG_DIR}\" -w \"${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}\"'"
            ./update_maintenance_wiki.sh -d -r "${RECONFIG_DIR}" -w "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}"
        else
            echo "  * Running: './update_maintenance_wiki.sh -r \"${RECONFIG_DIR}\" -w \"${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}\"'"
            ./update_maintenance_wiki.sh -r "${RECONFIG_DIR}" -w "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}"
            for file in "${RECONFIG_DIR}"/*_preview_changes.txt
            do
                mv "${file}" "$(echo "${file}" | sed "s/\\.txt\$/_${START_TIME}&/")"
            done 2>/dev/null || true
            RECONFIG_STOP_TIME="$(date +%s)"
            RECONFIG_ELAPSED=$((RECONFIG_STOP_TIME - START_TIME))
            RECONFIG_ELAPSED_DISPLAY=`show_time ${RECONFIG_ELAPSED}`
            IRC_MSG="Merge has finished in ${RECONFIG_ELAPSED_DISPLAY}. See http://bit.ly/reconfigs for details. Masters will reconfig themselves automatically on the hour."
            if [ "${FORCE_RECONFIG}" == '1' ]; then
                IRC_MSG="Reconfig has finished in ${RECONFIG_ELAPSED_DISPLAY}. See http://bit.ly/reconfigs for details."
            fi
            # Failing to update IRC is non-fatal.
            ./update_irc.sh "${IRC_MSG}" || true
        fi
    fi

    echo "  * Summary of changes:"
    cat "${RECONFIG_DIR}/${RECONFIG_UPDATE_FILE}" | sed 's/^/        /'
fi

# Manage foopies after everything else.
# No easy way to see if there are changes to the tools repo since there is no production branch
# and we do not tag it - so aside from running show_revision and then checking if every version
# is already on the latest commit, we should probably just run it regardless.
devices_json_url='http://hg.mozilla.org/build/tools/raw-file/tip/buildfarm/mobile/devices.json'
if [ "${PREPARE_ONLY}" != '0' ]; then
    echo "  * Preparing foopy update only; not running: '$(pwd)/manage_foopies.py' -f '${devices_json_url}' -j16 -H all show_revision update"
    echo "  * Preparing foopy update only; not running: '$(pwd)/manage_foopies.py' -f '${devices_json_url}' -j16 -H all show_revision"
else
    echo "  * Fabric log for foopies: '${RECONFIG_DIR}/manage_foopies-${START_TIME}.log'"
    echo "  * Running: '$(pwd)/manage_foopies.py' -f '${devices_json_url}' -j16 -H all show_revision update"
    ./manage_foopies.py -f "${devices_json_url}" -j16 -H all show_revision update >>"${RECONFIG_DIR}/manage_foopies-${START_TIME}.log" 2>&1
    echo "  * Running: '$(pwd)/manage_foopies.py' -f '${devices_json_url}' -j16 -H all show_revision"
    ./manage_foopies.py -f "${devices_json_url}" -j16 -H all show_revision >>"${RECONFIG_DIR}/manage_foopies-${START_TIME}.log" 2>&1
    echo "  * Foopies updated"
fi

echo "  * Directory '${RECONFIG_DIR}' contains artefacts from reconfig process"

STOP_TIME="$(date +%s)"
ELAPSED_TIME=$((STOP_TIME - START_TIME))
ELAPSED_DISPLAY=`show_time ${ELAPSED_TIME}`
echo "  * Finish timestamp: ${STOP_TIME}"
echo "  * Time taken: ${ELAPSED_DISPLAY}"