Difference between revisions of "Build Workshop 3: Build Hard With A Purpose/How Build Works"

From Eclipsepedia

Jump to: navigation, search
(Optional)
m
 
(9 intermediate revisions by 3 users not shown)
Line 12: Line 12:
 
# Path to a local checkout of the source (avoids checking out during build) (-localSourceCheckoutDir)
 
# Path to a local checkout of the source (avoids checking out during build) (-localSourceCheckoutDir)
 
# Nick should probably fill in the rest with all of the options he has
 
# Nick should probably fill in the rest with all of the options he has
 +
== Output ==
 +
# Master zip
 +
# SDK zip
 +
== Process ==
 +
=== Entry point:  [http://dev.eclipse.org/viewcvs/index.cgi/org.eclipse.dash/athena/org.eclipse.dash.commonbuilder/org.eclipse.dash.commonbuilder.releng/tools/scripts/start.sh?root=Technology_Project&view=markup start.sh] ===
 +
 +
==== Pre-build ====
 +
* build up options from command line input
 +
* small hacks for modeling projects
 +
* set up commonSriptsDir and configfile variables (cleanup?)
 +
 +
<source lang=bash>
 +
commonScriptsDir=$writableBuildRoot/build/org.eclipse.dash.common.releng/tools/scripts
 +
mkdir -p $commonScriptsDir
 +
cd $commonScriptsDir
 +
configfile=$commonScriptsDir/../../server.properties
 +
</source>
 +
 +
* set environment variables (could use a bit of cleanup)
 +
 +
<source lang=bash>
 +
export HOME=$writableBuildRoot
 +
if [ "x$javaHome" != "x" ]; then
 +
export JAVA_HOME=$javaHome;
 +
else # use default
 +
export JAVA_HOME=$($commonScriptsDir/readProperty.sh $configfile JAVA_HOME)
 +
javaHome="$JAVA_HOME"
 +
fi
 +
export ANT_HOME=$($commonScriptsDir/readProperty.sh $configfile ANT_HOME);
 +
ANT_BIN=$($commonScriptsDir/readProperty.sh $configfile ANT_BIN);
 +
if [ "x$ANT_BIN" != "x" ]; then
 +
export ANT=$ANT_BIN
 +
else
 +
export ANT=$ANT_HOME"/bin/ant";
 +
fi
 +
</source>
 +
 +
* look for PHP (I think we can remove this)
 +
 +
<source lang=bash>
 +
# get path to PHP interpreter
 +
if [ -x /usr/bin/php ]; then
 +
PHP=/usr/bin/php
 +
elif [ -x /usr/bin/php5 ]; then
 +
PHP=/usr/bin/php5
 +
elif [ -x /usr/bin/php4 ]; then
 +
PHP=/usr/bin/php4
 +
else
 +
PHP=php
 +
fi
 +
</source>
 +
 +
* set projectPath (remove "modeling"), hasProblem and hasFailed variables
 +
<source lang=bash>
 +
projectPath=modeling/$projectName/$subprojectName;
 +
 +
hasProblem="";
 +
hasFailed="";
 +
</source>
 +
 +
* checkBuildStatus() function (uses checkBuildStatus.php ... will need refactoring)
 +
* sendEmail() function (requires checkBuildStatus())
 +
* set downloadsDir and buildDir variables
 +
<source lang=bash>
 +
downloadsDir=$writableBuildRoot/build/downloads
 +
buildDir=$writableBuildRoot/build/$projectName/$subprojectName/downloads/drops/$version/$buildType$buildTimestamp
 +
</source>
 +
 +
* output the environment variables used by the build
 +
* if projRelengBranch not passed in, set it to $branch
 +
* for all dependency arguments passed in (-URL), download them unless they already exist in $downloadsDir
 +
** checkZipExists() uses checkZipExists.xml
 +
* create build directory and cd to it
 +
<source lang=bash>
 +
mkdir -p $buildDir/eclipse; cd $buildDir;
 +
</source>
 +
 +
* set up build.cfg and store some properties in it (including those already set in $tmpfile ... could probably echo to $buildcfg earlier)
 +
<source lang=bash>
 +
buildcfg="$buildDir/build.cfg";
 +
 +
cat $tmpfile >> $buildcfg;
 +
echo "" >> $buildcfg;
 +
rm -fr $tmpfile
 +
 +
echo "skipFetch=true" >> $buildcfg;
 +
echo "localSourceCheckoutDir=$localSourceCheckoutDir" >> $buildcfg
 +
</source>
 +
 +
* for each of org.eclipse.dash.common.releng, org.eclipse.$subjprojectName2.releng, and org.eclipse.releng.basebuilder, copy existing checkouts from either $writableBuildRoot/build, $writableBuildRoot/build/<item>_<branch> OR export from CVS
 +
** Example:
 +
** if $writableBuildRoot/build/org.eclipse.dash.common.releng exists, copy it to $buildDir/org.eclipse.dash.common.releng
 +
** else if $writableBuildRoot/build/org.eclipse.dash.common.releng_${commonRelengBranch} exists, copy it to $buildDir/org.eclipse.dash.common.releng
 +
** else if $buildDir/org.eclipse.dash.common.releng doesn't exist, export it from CVS
 +
<source lang=bash>
 +
# Defaults
 +
quietCVS=-Q
 +
commonRelengBranch="HEAD"
 +
cvs -d :pserver:anonymous@dev.eclipse.org:/cvsroot/technology $quietCVS ex -r $commonRelengBranch -d org.eclipse.dash.common.releng org.eclipse.dash/athena/org.eclipse.dash.commonbuilder/org.eclipse.dash.commonbuilder.releng"
 +
</source>
 +
 +
* cd to CBI tools/scripts directory in $buildDir
 +
* set up some more variables and append them to build.cfg
 +
<source lang=bash>
 +
relengBuilderDir=$buildDir/org.eclipse.$subprojectName2.releng
 +
relengCommonBuilderDir=$buildDir/org.eclipse.dash.common.releng
 +
relengBaseBuilderDir=$buildDir/org.eclipse.releng.basebuilder
 +
echo "java.home=$JAVA_HOME" >> $buildcfg;
 +
</source>
 +
 +
* set buildTag to either $mapfileTag or ${buildType}${buildTimestamp}
 +
* set baseLocation=$buildDir/eclipse ($buildDirEclipse) and append it to build.cfg
 +
* mkdir -p $buildDirEclipse $downloadsDir
 +
 +
* call [[#genBuildDetails.sh]]
 +
 +
==== Actual build ====
 +
* launch basebuilder's antRunner, calling $antTarget (defaults to "run") on $buildDir/org.eclipse.dash.common.releng/buildAll.xml with these arguments:
 +
** -DmapVersionTag=$buildTag
 +
** -DbuildType=$buildType
 +
** -DbuildID=$buildID
 +
** -Dtimestamp=$buildTimestamp
 +
** -DbuildDirectory=$buildDirEclipse
 +
** -DdownloadsDir=$downloadsDir
 +
** -DJAVA_HOME=$JAVA_HOME
 +
** and optionally (if $buildAlias is set)
 +
** -DbuildAlias=$buildAlias
 +
 +
==== Post-build ====
 +
 +
* call [[#getCompilerResults.sh]] on $buildDir/compilelogs => $buildDir/compilelogs/summary.txt
 +
 +
* if we've given a $depsFile, add the output of this build (the SDK.zip) into it
 +
<source lang=bash>
 +
if [[ $depsFile != "" ]]; then
 +
if [[ -f $depsFile ]]; then
 +
depNum=$(cat $depsFile | grep "$subprojectName=" | tail -1); depNum=${depNum%%$subprojectName=*};
 +
SDKURL=$(find $buildDir -maxdepth 1 -name "$projectName-$subprojectName-*SDK*.zip" | tail -1);
 +
# TODO: support non-standard SDK names (eg., GEF-SDK*.zip)
 +
if [[ $SDKURL != "" ]]; then
 +
SDKURL=${SDKURL##$buildDir}; SDKURL=${SDKURL##*/};
 +
SDKURL=http://$hostname/$projectPath/downloads/drops/$version/$buildType$buildTimestamp/$SDKURL;
 +
if [[ -w $depsFile ]]; then
 +
echo "$depNum$subprojectName=$SDKURL" >> $depsFile;
 +
echo "[promote] $SDKURL ($depNum$subprojectName) appended to $depsFile.";
 +
else
 +
echo "[promote] *** WARNING: File $depsFile is not writable. Add this manually:";
 +
echo "$depNum$subprojectName=$SDKURL"
 +
echo "[promote] ***";
 +
fi
 +
else
 +
echo "[start] *** WARNING: no SDK zip found in $buildDir. ***";
 +
fi
 +
else
 +
echo "[start] *** WARNING: cannot store SDK. File $depsFile does not exist. ***";
 +
fi
 +
fi
 +
</source>
 +
 +
* call checkBuildStatus()
 +
<source lang=bash>
 +
# should set noclean=1, hasProblem/hasFailed & cc: default email address
 +
# if there is a problem in the build
 +
checkBuildStatus()
 +
{
 +
projectNameActual=$projectName;
 +
checkBuildStatusURL="http://$hostname/modeling/build/checkBuildStatus.php?parent=$topprojectName&top=$projectName&project=$subprojectName&version=$version";
 +
checkBuildStatusURL=$checkBuildStatusURL"&buildID=$buildType$buildTimestamp";
 +
echo "[start] Checking build status from $checkBuildStatusURL ...";
 +
result=$(wget "$checkBuildStatusURL" -O - 2>/dev/null);
 +
hasProblem=$(echo $result | egrep "FAIL|ERROR| F| E");
 +
hasFailed=$(echo $result | egrep "FAIL| F");
 +
isUnknown=$(echo $result | egrep "UNKNOWN");
 +
if [[ $hasFailed ]]; then noclean=1; fi
 +
 +
# special case for emf/emft
 +
if [[ $isUnknown ]] && [[ $projectName == "emf" ]]; then
 +
projectName2="emft";
 +
checkBuildStatusURL="http://$hostname/modeling/build/checkBuildStatus.php?parent=$topprojectName&top=$projectName2&project=$subprojectName&version=$version";
 +
checkBuildStatusURL=$checkBuildStatusURL"&buildID=$buildType$buildTimestamp";
 +
echo "[start] Checking build status from $checkBuildStatusURL ...";
 +
result=$(wget "$checkBuildStatusURL" -O - 2>/dev/null);
 +
hasProblem=$(echo $result | egrep "FAIL|ERROR| F| E");
 +
hasFailed=$(echo $result | egrep "FAIL| F");
 +
isUnknown=$(echo $result | egrep "UNKNOWN");
 +
if [[ ! $isUnknown ]]; then
 +
projectNameActual=$projectName2;
 +
fi
 +
if [[ $hasFailed ]]; then noclean=1; fi
 +
fi
 +
if [[ $hasProblem ]]; then
 +
emailDefault=$($commonScriptsDir/readProperty.sh $configfile emailDefault);
 +
emailDefault=${emailDefault#*=> \"};
 +
emailDefault=${emailDefault%%\"*};
 +
if [[ ! $email ]]; then
 +
email=$emailDefault;
 +
elif [[ $email ]] && [[ $emailDefault ]]; then
 +
email=$email","$emailDefault;
 +
fi
 +
fi
 +
}
 +
</source>
 +
 +
* if there was a problem or we've requested an email, call sendEmail()
 +
<source lang=bash>
 +
# must call checkBuildStatus before sendEmail to ensure we have all the right email addresses listed (user-defined + default/backup)
 +
sendEmail ()
 +
{
 +
if [[ $email ]]; then
 +
$commonScriptsDir/executeCommand.sh "$PHP -q $commonScriptsDir/sendEmail.php \
 +
  -email $email -proj $projectNameActual -sub $subprojectName \
 +
  -branch $branch -version $version -buildID $buildType$buildTimestamp \
 +
  -hostname $hostname -parent $topprojectName";
 +
fi
 +
}
 +
</source>
 +
 +
* unless $noclean is set to 1 or the $buildDir has a file called "noclean" with a value of 1 in it
 +
<source lang=bash>
 +
rm -fr $buildDir/org.eclipse.dash.common.releng
 +
rm -fr $buildDir/org.eclipse.$subprojectName2.releng
 +
rm -fr $buildDir/org.eclipse.releng.basebuilder
 +
rm -fr $buildDir/eclipse
 +
rm -fr $buildDir/testing
 +
</source>
 +
 +
=== [http://dev.eclipse.org/viewcvs/index.cgi/org.eclipse.dash/athena/org.eclipse.dash.commonbuilder/org.eclipse.dash.commonbuilder.releng/tools/scripts/genBuildDetails.sh?root=Technology_Project&view=markup genBuildDetails.sh] ===
 +
 +
=== [http://dev.eclipse.org/viewcvs/index.cgi/org.eclipse.dash/athena/org.eclipse.dash.commonbuilder/org.eclipse.dash.commonbuilder.releng/tools/scripts/getCompilerResults.sh?root=Technology_Project&view=markup getCompilerResults.sh] ===
 +
<source lang=bash>
 +
# given a directory (./compilelogs)
 +
# recurse through all files and look for files containing
 +
#  975 problems (975 warnings)
 +
# generate a summary file which contains a count of problems, warnings, errors, and failures in the format:
 +
#  975P, 975W, 0E, 0F
 +
# or if none found, file is not created
 +
</source>
 +
 +
{{stub}}
 +
[[Category:Releng]] [[Category:Coordinated]] [[Category:Athena Common Build Workshops]]

Latest revision as of 13:52, 9 October 2009

Contents

[edit] How CBI Builds work

[edit] Input

[edit] Required

  1. Project ID (same as portal ex. tools.gef, technology.linuxtools) (-projectid)
  2. Version to use (-version)
  3. CVS Root for project's releng plugin (we will add SVN support soon) <-- could perhaps come from portal in future (-projRelengRoot)
  4. CVS Path for project's releng plugin (we will add SVN support soon) <-- could perhaps come from portal in future (-projRelengPath)
  5. URLs of dependencies (ex. http://download.eclipse.org/eclipse/downloads/drops/R-3.4-200806172000/eclipse-SDK-3.4-linux-gtk.tar.gz) (we will automate this more in the future) (-URL)

[edit] Optional

  1. CVS branch of org.eclipse.releng.basebuilder (-basebuilderBranch)
  2. JAVA_HOME (-javaHome)
  3. Path to a local checkout of the source (avoids checking out during build) (-localSourceCheckoutDir)
  4. Nick should probably fill in the rest with all of the options he has

[edit] Output

  1. Master zip
  2. SDK zip

[edit] Process

[edit] Entry point: start.sh

[edit] Pre-build

  • build up options from command line input
  • small hacks for modeling projects
  • set up commonSriptsDir and configfile variables (cleanup?)
commonScriptsDir=$writableBuildRoot/build/org.eclipse.dash.common.releng/tools/scripts
mkdir -p $commonScriptsDir
cd $commonScriptsDir
configfile=$commonScriptsDir/../../server.properties
  • set environment variables (could use a bit of cleanup)
export HOME=$writableBuildRoot
if [ "x$javaHome" != "x" ]; then
	export JAVA_HOME=$javaHome;
else # use default
	export JAVA_HOME=$($commonScriptsDir/readProperty.sh $configfile JAVA_HOME)
	javaHome="$JAVA_HOME"
fi
export ANT_HOME=$($commonScriptsDir/readProperty.sh $configfile ANT_HOME);
ANT_BIN=$($commonScriptsDir/readProperty.sh $configfile ANT_BIN);
if [ "x$ANT_BIN" != "x" ]; then
	export ANT=$ANT_BIN
else
	export ANT=$ANT_HOME"/bin/ant";
fi
  • look for PHP (I think we can remove this)
# get path to PHP interpreter
if [ -x /usr/bin/php ]; then
	PHP=/usr/bin/php
elif [ -x /usr/bin/php5 ]; then
	PHP=/usr/bin/php5
elif [ -x /usr/bin/php4 ]; then
	PHP=/usr/bin/php4
else
	PHP=php
fi
  • set projectPath (remove "modeling"), hasProblem and hasFailed variables
projectPath=modeling/$projectName/$subprojectName;
 
hasProblem="";
hasFailed="";
  • checkBuildStatus() function (uses checkBuildStatus.php ... will need refactoring)
  • sendEmail() function (requires checkBuildStatus())
  • set downloadsDir and buildDir variables
downloadsDir=$writableBuildRoot/build/downloads
buildDir=$writableBuildRoot/build/$projectName/$subprojectName/downloads/drops/$version/$buildType$buildTimestamp
  • output the environment variables used by the build
  • if projRelengBranch not passed in, set it to $branch
  • for all dependency arguments passed in (-URL), download them unless they already exist in $downloadsDir
    • checkZipExists() uses checkZipExists.xml
  • create build directory and cd to it
mkdir -p $buildDir/eclipse; cd $buildDir;
  • set up build.cfg and store some properties in it (including those already set in $tmpfile ... could probably echo to $buildcfg earlier)
buildcfg="$buildDir/build.cfg";
 
cat $tmpfile >> $buildcfg;
echo "" >> $buildcfg;
rm -fr $tmpfile
 
echo "skipFetch=true" >> $buildcfg;
echo "localSourceCheckoutDir=$localSourceCheckoutDir" >> $buildcfg
  • for each of org.eclipse.dash.common.releng, org.eclipse.$subjprojectName2.releng, and org.eclipse.releng.basebuilder, copy existing checkouts from either $writableBuildRoot/build, $writableBuildRoot/build/<item>_<branch> OR export from CVS
    • Example:
    • if $writableBuildRoot/build/org.eclipse.dash.common.releng exists, copy it to $buildDir/org.eclipse.dash.common.releng
    • else if $writableBuildRoot/build/org.eclipse.dash.common.releng_${commonRelengBranch} exists, copy it to $buildDir/org.eclipse.dash.common.releng
    • else if $buildDir/org.eclipse.dash.common.releng doesn't exist, export it from CVS
# Defaults
quietCVS=-Q
commonRelengBranch="HEAD"
cvs -d :pserver:anonymous@dev.eclipse.org:/cvsroot/technology $quietCVS ex -r $commonRelengBranch -d org.eclipse.dash.common.releng org.eclipse.dash/athena/org.eclipse.dash.commonbuilder/org.eclipse.dash.commonbuilder.releng"
  • cd to CBI tools/scripts directory in $buildDir
  • set up some more variables and append them to build.cfg
relengBuilderDir=$buildDir/org.eclipse.$subprojectName2.releng
relengCommonBuilderDir=$buildDir/org.eclipse.dash.common.releng
relengBaseBuilderDir=$buildDir/org.eclipse.releng.basebuilder
echo "java.home=$JAVA_HOME" >> $buildcfg;
  • set buildTag to either $mapfileTag or ${buildType}${buildTimestamp}
  • set baseLocation=$buildDir/eclipse ($buildDirEclipse) and append it to build.cfg
  • mkdir -p $buildDirEclipse $downloadsDir

[edit] Actual build

  • launch basebuilder's antRunner, calling $antTarget (defaults to "run") on $buildDir/org.eclipse.dash.common.releng/buildAll.xml with these arguments:
    • -DmapVersionTag=$buildTag
    • -DbuildType=$buildType
    • -DbuildID=$buildID
    • -Dtimestamp=$buildTimestamp
    • -DbuildDirectory=$buildDirEclipse
    • -DdownloadsDir=$downloadsDir
    • -DJAVA_HOME=$JAVA_HOME
    • and optionally (if $buildAlias is set)
    • -DbuildAlias=$buildAlias

[edit] Post-build

  • if we've given a $depsFile, add the output of this build (the SDK.zip) into it
if [[ $depsFile != "" ]]; then
	if [[ -f $depsFile ]]; then
		depNum=$(cat $depsFile | grep "$subprojectName=" | tail -1); depNum=${depNum%%$subprojectName=*};
		SDKURL=$(find $buildDir -maxdepth 1 -name "$projectName-$subprojectName-*SDK*.zip" | tail -1);
		# TODO: support non-standard SDK names (eg., GEF-SDK*.zip)
		if [[ $SDKURL != "" ]]; then 
			SDKURL=${SDKURL##$buildDir}; SDKURL=${SDKURL##*/}; 
			SDKURL=http://$hostname/$projectPath/downloads/drops/$version/$buildType$buildTimestamp/$SDKURL; 
			if [[ -w $depsFile ]]; then 
				echo "$depNum$subprojectName=$SDKURL" >> $depsFile;
				echo "[promote] $SDKURL ($depNum$subprojectName) appended to $depsFile.";
			else
				echo "[promote] *** WARNING: File $depsFile is not writable. Add this manually:";
				echo "$depNum$subprojectName=$SDKURL"
				echo "[promote] ***";
			fi
		else
			echo "[start] *** WARNING: no SDK zip found in $buildDir. ***"; 
		fi
	else
		echo "[start] *** WARNING: cannot store SDK. File $depsFile does not exist. ***";
	fi
fi
  • call checkBuildStatus()
# should set noclean=1, hasProblem/hasFailed & cc: default email address
# if there is a problem in the build 
checkBuildStatus()
{
	projectNameActual=$projectName;
	checkBuildStatusURL="http://$hostname/modeling/build/checkBuildStatus.php?parent=$topprojectName&top=$projectName&project=$subprojectName&version=$version";
	checkBuildStatusURL=$checkBuildStatusURL"&buildID=$buildType$buildTimestamp";
	echo "[start] Checking build status from $checkBuildStatusURL ..."; 
	result=$(wget "$checkBuildStatusURL" -O - 2>/dev/null);
	hasProblem=$(echo $result | egrep "FAIL|ERROR| F| E"); 
	hasFailed=$(echo $result | egrep "FAIL| F"); 
	isUnknown=$(echo $result | egrep "UNKNOWN"); 
	if [[ $hasFailed ]]; then noclean=1; fi
 
	# special case for emf/emft
	if [[ $isUnknown ]] && [[ $projectName == "emf" ]]; then 
		projectName2="emft";
		checkBuildStatusURL="http://$hostname/modeling/build/checkBuildStatus.php?parent=$topprojectName&top=$projectName2&project=$subprojectName&version=$version";
		checkBuildStatusURL=$checkBuildStatusURL"&buildID=$buildType$buildTimestamp"; 
		echo "[start] Checking build status from $checkBuildStatusURL ..."; 
		result=$(wget "$checkBuildStatusURL" -O - 2>/dev/null);
		hasProblem=$(echo $result | egrep "FAIL|ERROR| F| E"); 
		hasFailed=$(echo $result | egrep "FAIL| F"); 
		isUnknown=$(echo $result | egrep "UNKNOWN"); 
		if [[ ! $isUnknown ]]; then
			projectNameActual=$projectName2;
		fi
		if [[ $hasFailed ]]; then noclean=1; fi
	fi
	if [[ $hasProblem ]]; then
		emailDefault=$($commonScriptsDir/readProperty.sh $configfile emailDefault); 
		emailDefault=${emailDefault#*=> \"}; 
		emailDefault=${emailDefault%%\"*};
		 if [[ ! $email ]]; then 
			email=$emailDefault;
		elif [[ $email ]] && [[ $emailDefault ]]; then
			email=$email","$emailDefault;
		fi
	fi
}
  • if there was a problem or we've requested an email, call sendEmail()
# must call checkBuildStatus before sendEmail to ensure we have all the right email addresses listed (user-defined + default/backup)
sendEmail () 
{
	if [[ $email ]]; then
		$commonScriptsDir/executeCommand.sh "$PHP -q $commonScriptsDir/sendEmail.php \
		  -email $email -proj $projectNameActual -sub $subprojectName \
		  -branch $branch -version $version -buildID $buildType$buildTimestamp \
		  -hostname $hostname -parent $topprojectName";
	fi
}
  • unless $noclean is set to 1 or the $buildDir has a file called "noclean" with a value of 1 in it
rm -fr $buildDir/org.eclipse.dash.common.releng
rm -fr $buildDir/org.eclipse.$subprojectName2.releng
rm -fr $buildDir/org.eclipse.releng.basebuilder
rm -fr $buildDir/eclipse
rm -fr $buildDir/testing

[edit] genBuildDetails.sh

[edit] getCompilerResults.sh

# given a directory (./compilelogs)
# recurse through all files and look for files containing 
#   975 problems (975 warnings)
# generate a summary file which contains a count of problems, warnings, errors, and failures in the format:
#   975P, 975W, 0E, 0F
# or if none found, file is not created