Blame get_sources.sh

Karanbir Singh afa935
#!/bin/bash
Karanbir Singh afa935
#
Karanbir Singh afa935
# Might want to drop this in ~/bin/ and chmod u+x it
Pat Riehecky 645621
#
Pat Riehecky 645621
Pat Riehecky 645621
#  Initial Author: Karanbir Singh <kbsingh@centos.org>
Pat Riehecky 645621
#         Updates:
Pat Riehecky 645621
#                  Mike McLean <mikem@redhat.com>
Pat Riehecky 645621
#                  Pat Riehecky <riehecky@fnal.gov>
Tyler Parsons d96c00
#                  Tyler Parsons <tparsons@fnal.gov>
Tuomo Soini bd83a7
#                  Tuomo Soini <tis@foobar.fi>
Pat Riehecky 645621
Pat Riehecky 645621
Pat Riehecky 645621
#####################################################################
Pat Riehecky 645621
usage() {
Pat Riehecky 645621
    echo ''                                               >&2
Pat Riehecky 645621
    echo "$0 [-hcq] [-b branch] [--surl url]"             >&2
Pat Riehecky 645621
    echo ''                                               >&2
Pat Riehecky 645621
    echo 'Script to parse the non-text sources metadata file'   >&2
Pat Riehecky 645621
    echo ' and download the required files from the lookaside'  >&2
Pat Riehecky 645621
    echo ' cache.'                                              >&2
Pat Riehecky 645621
    echo ''                                                     >&2
Pat Riehecky 645621
    echo 'PLEASE NOTE: this script is non-destructive, it wont' >&2
Pat Riehecky 645621
    echo ' replace files that already exist, regardless of'     >&2
Pat Riehecky 645621
    echo ' their state, allowing you to have work-in-progress'  >&2
Pat Riehecky 645621
    echo ' content that wont get overwritten.'                  >&2
Pat Riehecky 645621
    echo ''                                                     >&2
Pat Riehecky 645621
    echo 'You need to run this from inside a sources git repo'  >&2
Pat Riehecky 645621
    echo ''                                               >&2
Pat Riehecky 645621
    echo ' -h: This help message'                         >&2
Pat Riehecky 645621
    echo ''                                               >&2
Pat Riehecky 645621
    echo "  $0 -b c7"                                     >&2
Pat Riehecky 645621
    echo "  $0 -q -b c7"                                  >&2
Pat Riehecky 645621
    echo "  $0 -c -b remotes/origin/c7"                   >&2
Pat Riehecky 645621
    echo "  $0 -c -b c7 --surl '$SURL'"                   >&2
Pat Riehecky 645621
    echo "  $0"                                           >&2
Pat Riehecky 645621
    exit 1
Pat Riehecky 645621
}
Karanbir Singh afa935
Pat Riehecky 645621
#####################################################################
Karanbir Singh afa935
696c29
SURL="https://git.centos.org/sources"
Pat Riehecky 645621
Pat Riehecky 645621
QUIET=0
Pat Riehecky 645621
BRANCH=''
Pat Riehecky 645621
CHECK=0
Pat Riehecky 645621
Pat Riehecky 645621
# for setting any overrides, such as SURL, default BRANCH, or force CHECK
Pat Riehecky 1842fe
if [ -f /etc/centos-git-common ]; then
Pat Riehecky 1842fe
  . /etc/centos-git-common
Pat Riehecky 1842fe
fi
Pat Riehecky 1842fe
Pat Riehecky 645621
#####################################################################
Pat Riehecky 645621
# setup args in the right order for making getopt evaluation
Pat Riehecky 645621
# nice and easy.  You'll need to read the manpages for more info
Pat Riehecky 645621
# utilizing 'while' construct rather than 'for arg' to avoid unnecessary
Pat Riehecky 645621
# shifting of program args
Pat Riehecky 645621
args=$(getopt -o hcqb: -l surl: -- "$@")
Pat Riehecky 645621
eval set -- "$args"
Pat Riehecky 645621
Pat Riehecky 645621
while [[ 0 -eq 0 ]]; do
Pat Riehecky 645621
    case $1 in
Pat Riehecky 645621
        -- )
Pat Riehecky 645621
            # end of getopt args, shift off the -- and get out of the loop
Pat Riehecky 645621
            shift
Pat Riehecky 645621
            break
Pat Riehecky 645621
           ;;
Pat Riehecky 645621
         -c )
Pat Riehecky 645621
            # verify the sha1sum of the downloaded file
Pat Riehecky 645621
            CHECK=1
Pat Riehecky 645621
            shift
Pat Riehecky 645621
           ;;
Pat Riehecky 645621
         -q )
Pat Riehecky 645621
            # suppress warnings
Pat Riehecky 645621
            QUIET=1
Pat Riehecky 645621
            shift
Pat Riehecky 645621
           ;;
Pat Riehecky 645621
         -b )
Pat Riehecky 645621
            # Check this particular branch 
Pat Riehecky 645621
            BRANCH=$2
Pat Riehecky 645621
            shift
Pat Riehecky 645621
            shift
Pat Riehecky 645621
           ;;
Pat Riehecky 645621
         --surl )
Pat Riehecky 645621
            # override sources url
Pat Riehecky 645621
            SURL=$2
Pat Riehecky 645621
            shift
Pat Riehecky 645621
            shift
Pat Riehecky 645621
           ;;
Pat Riehecky 645621
         -h )
Pat Riehecky 645621
            # get help
Pat Riehecky 645621
            usage
Pat Riehecky 645621
           ;;
Pat Riehecky 645621
    esac
Mike McLean c61c1c
done
Mike McLean c61c1c
Pat Riehecky 645621
# set curl options this way so defaults can be set in /etc/centos-git-common
Pat Riehecky 645621
# across multiple scripts
Pat Riehecky 645621
if [[ ${QUIET} -eq 1 ]]; then
Pat Riehecky 645621
    QUIET='--silent'
Pat Riehecky 645621
else
Pat Riehecky 645621
    QUIET=''
Pat Riehecky 645621
fi
Pat Riehecky 645621
Pat Riehecky 645621
which git >/dev/null 2>&1
Pat Riehecky 645621
if [[ $? -ne 0 ]]; then
Pat Riehecky 645621
    echo 'You need git in PATH' >&2
Pat Riehecky 645621
    exit 1
Pat Riehecky 645621
fi
Pat Riehecky 645621
Pat Riehecky 645621
which curl >/dev/null 2>&1
Pat Riehecky 645621
if [[ $? -ne 0 ]]; then
Pat Riehecky 645621
    echo 'You need curl in PATH' >&2
Pat Riehecky 645621
    exit 1
Pat Riehecky 645621
fi
Pat Riehecky 645621
Tyler Parsons d96c00
# should go into a function section at some point
Tyler Parsons d96c00
weakHashDetection () {
Tyler Parsons d96c00
  strHash=${1};
Tyler Parsons d96c00
  case $((`echo ${strHash}|wc -m` - 1 )) in
Tyler Parsons d96c00
    128)
Tyler Parsons d96c00
      hashBin='sha512sum'
Tyler Parsons d96c00
      ;;
Tyler Parsons d96c00
    64)
Tyler Parsons d96c00
      hashBin='sha256sum'
Tyler Parsons d96c00
      ;;
Tyler Parsons d96c00
    40)
Tyler Parsons d96c00
      hashBin='sha1sum'
Tyler Parsons d96c00
      ;;
Tyler Parsons d96c00
    32)
Tyler Parsons d96c00
      hashBin='md5sum'
Tyler Parsons d96c00
      ;;
Tyler Parsons d96c00
    *)
Tyler Parsons d96c00
      hashBin='unknown'
Tyler Parsons d96c00
      ;;
Tyler Parsons d96c00
  esac
Tyler Parsons d96c00
  echo ${hashBin};
Tyler Parsons d96c00
}
Pat Riehecky 645621
Mike McLean c61c1c
# check metadata file and extract package name
Mike McLean c61c1c
shopt -s nullglob
Mike McLean da2660
set -- .*.metadata
Mike McLean da2660
if (( $# == 0 ))
Mike McLean c61c1c
then
Pat Riehecky 7520e4
    echo 'Missing metadata. Please run from inside a sources git repo' >&2
Mike McLean c61c1c
    exit 1
Mike McLean da2660
elif (( $# > 1 ))
Mike McLean da2660
then
Mike McLean da2660
    echo "Warning: multiple metadata files found. Using $1"
Mike McLean c61c1c
fi
Mike McLean da2660
meta=$1
Mike McLean da2660
pn=${meta%.metadata}
Mike McLean da2660
pn=${pn#.}
Mike McLean c61c1c
Mike McLean c61c1c
if [ ! -d .git ] || [ ! -d SPECS ]; then
Pat Riehecky 7520e4
  echo 'You need to run this from inside a sources git repo' >&2
Karanbir Singh afa935
  exit 1
Karanbir Singh afa935
fi
Pat Riehecky f3a920
mkdir -p SOURCES
Mike McLean c61c1c
Mike McLean c61c1c
# sort out our branch
Mike McLean c61c1c
if [ -n "$BRANCH" ]
Mike McLean c61c1c
then
Mike McLean c61c1c
  branches=("$BRANCH")
Mike McLean c61c1c
else
Mike McLean c61c1c
  # generate a list of all branches containing current HEAD
Mike McLean c61c1c
  branches=()
Mike McLean c61c1c
  while IFS='' read -r line
Mike McLean c61c1c
  do
Mike McLean c61c1c
    # input from: git branch --contains HEAD
Mike McLean c61c1c
    branch="${line:2}"
Mike McLean c61c1c
    if [[ "$branch" =~ "detached from" ]]
Mike McLean c61c1c
    then
Mike McLean c61c1c
      # ignore detached heads
Mike McLean c61c1c
      continue
Mike McLean c61c1c
    fi
Mike McLean c61c1c
    if [ ".${line:0:1}" = ".*" ]
Mike McLean c61c1c
    then
Mike McLean c61c1c
      # current branch, put it first
Mike McLean c61c1c
      branches=("$branch" "${branches[@]}")
Mike McLean c61c1c
    else
Mike McLean c61c1c
      branches=("${branches[@]}" "$branch")
Mike McLean c61c1c
    fi
Mike McLean c61c1c
  done <<< "$(git branch --contains HEAD)"
Mike McLean c61c1c
fi
Mike McLean c61c1c
while read -r fsha fname ; do
Mike McLean c61c1c
  if [ ".${fsha}" = ".da39a3ee5e6b4b0d3255bfef95601890afd80709" ]; then
Karanbir Singh afa935
    # zero byte file
Pat Riehecky a4a7af
    touch ${fname}
Karanbir Singh afa935
  else
Tyler Parsons d96c00
    if [ ${CHECK} -eq 1 ]; then
Tyler Parsons d96c00
      hashType=$(weakHashDetection ${fsha})
Tyler Parsons d96c00
      if [ "${hashType}" == "unknown" ]; then
Tyler Parsons d96c00
        echo 'Failure: Hash type unknown.' >&2
Tyler Parsons d96c00
        exit 1;
Tyler Parsons d96c00
      else
Tyler Parsons d96c00
        which ${hashType} >/dev/null 2>&1
Tyler Parsons d96c00
        if [[ $? -ne 0 ]]; then
Tyler Parsons d96c00
          echo "Failure: You need ${hashType} in PATH." >&2
Tyler Parsons d96c00
          exit 1;
Tyler Parsons d96c00
        fi
Tyler Parsons d96c00
      fi
Tyler Parsons d96c00
    fi
Tuomo Soini bd83a7
    if [ -e ${fname} -a ${CHECK} -eq 1 ]; then
Tuomo Soini bd83a7
	# check hash sum and force download if wrong
Tuomo Soini bd83a7
        downsum=$(${hashType} ${fname} | awk '{print $1}')
Tuomo Soini bd83a7
        if [ "${fsha}" != "${downsum}" ]; then
Tuomo Soini bd83a7
            rm -f ${fname}
Tuomo Soini bd83a7
        fi
Tuomo Soini bd83a7
    fi
Mike McLean c61c1c
    if [ ! -e "${fname}" ]; then
Mike McLean c61c1c
      for br in "${branches[@]}"
Mike McLean c61c1c
      do
Pat Riehecky 645621
        br=$(echo ${br}| sed -e s'|remotes/origin/||')
Colin Walters 1ad349
        url="${SURL}/${pn}/${br}/${fsha}"
Colin Walters 1ad349
        echo "Retrieving ${url}"
Colin Walters 1ad349
        curl -L ${QUIET} -f "${url}" -o "${fname}" && break
Mike McLean c61c1c
      done
Karanbir Singh afa935
    else
Karanbir Singh afa935
      echo "${fname} exists. skipping"
Karanbir Singh afa935
    fi
Pat Riehecky 061172
    if [ ${CHECK} -eq 1 ]; then
Tyler Parsons d96c00
        downsum=$(${hashType} ${fname} | awk '{print $1}')
Tyler Parsons d96c00
        if [ "${fsha}" != "${downsum}" ]; then
Pat Riehecky 061172
            rm -f ${fname}
Tyler Parsons d96c00
            echo "Failure: ${fname} hash does not match hash from the .metadata file" >&2
Tyler Parsons d96c00
            exit 1;
Pat Riehecky 061172
        fi
Pat Riehecky 061172
    fi
Karanbir Singh afa935
  fi
Mike McLean c61c1c
done < "${meta}"