test-release.sh: Update to fetch source from GitHub

Summary:
This also changes the test-release.sh script to build using the monorepo
layout instead of copying sub-projects into llvm/tools or llvm/projects.

Reviewers: jdoerfert, hans

Reviewed By: hans

Subscribers: hans, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D70353

llvm/utils/release/test-release.sh

@@ -43,6 +43,7 @@ do_polly="yes"
 BuildDir="`pwd`"
 ExtraConfigureFlags=""
 ExportBranch=""
+git_ref=""
 
 function usage() {
     echo "usage: `basename $0` -release X.Y.Z -rc NUM [OPTIONS]"
@@ -60,8 +61,7 @@ function usage() {
     echo " -use-gzip            Use gzip instead of xz."
     echo " -use-ninja           Use ninja instead of make/gmake."
     echo " -configure-flags FLAGS  Extra flags to pass to the configure step."
-    echo " -svn-path DIR        Use the specified DIR instead of a release."
+    echo " -git-ref sha         Use the specified git ref for testing instead of a release."
-    echo "                      For example -svn-path trunk or -svn-path branches/release_37"
     echo " -no-rt               Disable check-out & build Compiler-RT"
     echo " -no-libs             Disable check-out & build libcxx/libcxxabi/libunwind"
     echo " -no-libcxxabi        Disable check-out & build libcxxabi"
@@ -88,13 +88,14 @@ while [ $# -gt 0 ]; do
         -final | --final )
             RC=final
             ;;
-        -svn-path | --svn-path )
+        -git-ref | --git-ref )
             shift
             Release="test"
             Release_no_dot="test"
             ExportBranch="$1"
             RC="`echo $ExportBranch | sed -e 's,/,_,g'`"
-            echo "WARNING: Using the branch $ExportBranch instead of a release tag"
+            git_ref="$1"
+            echo "WARNING: Using the ref $git_ref instead of a release tag"
             echo "         This is intended to aid new packagers in trialing "
             echo "         builds without requiring a tag to be created first"
             ;;
@@ -196,6 +197,17 @@ if [ -z "$Triple" ]; then
     exit 1
 fi
 
+if [ "$Release" != "test" ]; then
+  if [ -n "$git_ref" ]; then
+    echo "error: can't specify both -release and -git-ref"
+    exit 1
+  fi
+  git_ref=llvmorg-$Release
+  if [ "$RC" != "final" ]; then
+    git_ref="$git_ref-$RC"
+  fi
+fi
+
 # Figure out how many make processes to run.
 if [ -z "$NumJobs" ]; then
     NumJobs=`sysctl -n hw.activecpu 2> /dev/null || true`
@@ -211,7 +223,7 @@ if [ -z "$NumJobs" ]; then
 fi
 
 # Projects list
-projects="llvm cfe clang-tools-extra"
+projects="llvm clang clang-tools-extra"
 if [ $do_rt = "yes" ]; then
   projects="$projects compiler-rt"
 fi
@@ -288,60 +300,37 @@ fi
 
 check_program_exists ${MAKE}
 
-# Make sure that the URLs are valid.
-function check_valid_urls() {
-    for proj in $projects ; do
-        echo "# Validating $proj SVN URL"
-
-        if ! svn ls $Base_url/$proj/$ExportBranch > /dev/null 2>&1 ; then
-            echo "$proj does not have a $ExportBranch branch/tag!"
-            exit 1
-        fi
-    done
-}
-
 # Export sources to the build directory.
 function export_sources() {
-    check_valid_urls
+  SrcDir=$BuildDir/llvm-project
+  mkdir -p $SrcDir
+  echo "# Using git ref: $git_ref"
 
-    for proj in $projects ; do
+  # GitHub allows you to download a tarball of any commit using the URL:
-        case $proj in
+  # https://github.com/$organization/$repo/archive/$ref.tar.gz
-        llvm)
+  curl -L https://github.com/llvm/llvm-project/archive/$git_ref.tar.gz | \
-            projsrc=$proj.src
+    tar -C $SrcDir --strip-components=1 -xzf -
-            ;;
-        cfe)
-            projsrc=llvm.src/tools/clang
-            ;;
-        lld|lldb|polly)
-            projsrc=llvm.src/tools/$proj
-            ;;
-        clang-tools-extra)
-            projsrc=llvm.src/tools/clang/tools/extra
-            ;;
-        compiler-rt|libcxx|libcxxabi|libunwind|openmp)
-            projsrc=llvm.src/projects/$proj
-            ;;
-        test-suite)
-            projsrc=$proj.src
-            ;;
-        *)
-            echo "error: unknown project $proj"
-            exit 1
-            ;;
-        esac
 
-        if [ -d $projsrc ]; then
+  if [ "$do_test_suite" = "yes" ]; then
-          echo "# Reusing $proj $Release-$RC sources in $projsrc"
+    TestSuiteSrcDir=$BuildDir/llvm-test-suite
-          continue
+    mkdir -p $TestSuiteSrcDir
-        fi
-        echo "# Exporting $proj $Release-$RC sources to $projsrc"
-        if ! svn export -q $Base_url/$proj/$ExportBranch $projsrc ; then
-            echo "error: failed to export $proj project"
-            exit 1
-        fi
-    done
 
-    cd $BuildDir
+    # We can only use named refs, like branches and tags, that exist in
+    # both the llvm-project and test-suite repos if we want to run the
+    # test suite.
+    # If the test-suite fails to download assume we are using a ref that
+    # doesn't exist in the test suite and disable it.
+    set +e
+    curl -L https://github.com/llvm/test-suite/archive/$git_ref.tar.gz | \
+      tar -C $TestSuiteSrcDir --strip-components=1 -xzf -
+    if [ $? -ne -0 ]; then
+      echo "$git_ref not found in test-suite repo, test-suite disabled."
+      do_test_suite="no"
+    fi
+    set -e
+  fi
+
+  cd $BuildDir
 }
 
 function configure_llvmCore() {
@@ -369,6 +358,7 @@ function configure_llvmCore() {
             ;;
     esac
 
+    project_list=${projects// /;}
     echo "# Using C compiler: $c_compiler"
     echo "# Using C++ compiler: $cxx_compiler"
 
@@ -378,12 +368,14 @@ function configure_llvmCore() {
     echo "#" env CC="$c_compiler" CXX="$cxx_compiler" \
         cmake -G "$generator" \
         -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
-        $ExtraConfigureFlags $BuildDir/llvm.src \
+        -DLLVM_ENABLE_PROJECTS="$project_list" \
+        $ExtraConfigureFlags $BuildDir/llvm-project/llvm \
         2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
     env CC="$c_compiler" CXX="$cxx_compiler" \
         cmake -G "$generator" \
         -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
-        $ExtraConfigureFlags $BuildDir/llvm.src \
+        -DLLVM_ENABLE_PROJECTS="$project_list" \
+        $ExtraConfigureFlags $BuildDir/llvm-project/llvm \
         2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
 
     cd $BuildDir
@@ -494,10 +486,10 @@ if [ $do_test_suite = "yes" ]; then
   SandboxDir="$BuildDir/sandbox"
   Lit=$SandboxDir/bin/lit
   TestSuiteBuildDir="$BuildDir/test-suite-build"
-  TestSuiteSrcDir="$BuildDir/test-suite.src"
+  TestSuiteSrcDir="$BuildDir/llvm-test-suite"
 
   virtualenv $SandboxDir
-  $SandboxDir/bin/python $BuildDir/llvm.src/utils/lit/setup.py install
+  $SandboxDir/bin/python $BuildDir/llvm-project/llvm/utils/lit/setup.py install
   mkdir -p $TestSuiteBuildDir
 fi