[Clang][OpenMP Offload] Add new tool for wrapping offload device binaries

This patch removes the remaining part of the OpenMP offload linker scripts which was used for inserting device binaries into the output linked binary. Device binaries are now inserted into the host binary with a help of the wrapper bit-code file which contains device binaries as data. Wrapper bit-code file is dynamically created by the clang driver with a help of new tool clang-offload-wrapper which takes device binaries as input and produces bit-code file with required contents. Wrapper bit-code is then compiled to an object and resulting object is appended to the host linking by the clang driver.

This is the second part of the patch for eliminating OpenMP linker script (please see https://reviews.llvm.org/D64943).

Differential Revision: https://reviews.llvm.org/D68166

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@374219 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 155 insertions, 120 deletions

diff --git a/test/Driver/clang-offload-wrapper.c b/test/Driver/clang-offload-wrapper.c
new file mode 100644
index 0000000000..542930bc3c
--- /dev/null
+++ b/test/Driver/clang-offload-wrapper.c
@@ -0,0 +1,29 @@
+// REQUIRES: x86-registered-target
+
+//
+// Check help message.
+//
+// RUN: clang-offload-wrapper --help | FileCheck %s --check-prefix CHECK-HELP
+// CHECK-HELP: {{.*}}OVERVIEW: A tool to create a wrapper bitcode for offload target binaries. Takes offload
+// CHECK-HELP: {{.*}}target binaries as input and produces bitcode file containing target binaries packaged
+// CHECK-HELP: {{.*}}as data.
+// CHECK-HELP: {{.*}}USAGE: clang-offload-wrapper [options] <input files>
+// CHECK-HELP: {{.*}}  -o=<filename>               - Output filename
+// CHECK-HELP: {{.*}}  --offload-targets=<triples> - Comma-separated list of device target triples
+// CHECK-HELP: {{.*}}  --target=<triple>           - Target triple for the output module
+
+//
+// Generate a file to wrap.
+//
+// RUN: echo 'Content of device file' > %t.tgt
+
+//
+// Check bitcode produced by the wrapper tool.
+//
+// RUN: clang-offload-wrapper -target=x86_64-pc-linux-gnu -offload-targets=powerpc64le-ibm-linux-gnu -o %t.wrapper.bc %t.tgt
+// RUN: llvm-dis %t.wrapper.bc -o - | FileCheck %s --check-prefix CHECK-IR
+
+// CHECK-IR: target triple = "x86_64-pc-linux-gnu"
+
+// CHECK-IR: @.omp_offloading.img_start.powerpc64le-ibm-linux-gnu = hidden unnamed_addr constant [{{[0-9]+}} x i8] c"Content of device file{{.+}}", section ".omp_offloading.powerpc64le-ibm-linux-gnu"
+// CHECK-IR: @.omp_offloading.img_end.powerpc64le-ibm-linux-gnu = hidden unnamed_addr constant [0 x i8] zeroinitializer, section ".omp_offloading.powerpc64le-ibm-linux-gnu"
diff --git a/test/Driver/openmp-offload-gpu.c b/test/Driver/openmp-offload-gpu.c
index 3d2ac4525f..dc4dbd1f37 100644
--- a/test/Driver/openmp-offload-gpu.c
+++ b/test/Driver/openmp-offload-gpu.c
@@ -55,7 +55,7 @@
 // RUN:          -fopenmp-targets=nvptx64-nvidia-cuda %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=CHK-CUBIN-NVLINK %s
 
-// CHK-CUBIN-NVLINK: clang{{.*}}" "-o" "[[PTX:.*\.s]]"
+// CHK-CUBIN-NVLINK: clang{{.*}}" {{.*}}"-fopenmp-is-device" {{.*}}"-o" "[[PTX:.*\.s]]"
 // CHK-CUBIN-NVLINK-NEXT: ptxas{{.*}}" "--output-file" "[[CUBIN:.*\.cubin]]" {{.*}}"[[PTX]]"
 // CHK-CUBIN-NVLINK-NEXT: nvlink{{.*}}" {{.*}}"[[CUBIN]]"
 
diff --git a/test/Driver/openmp-offload.c b/test/Driver/openmp-offload.c
index 0ba3c43263..292818b84f 100644
--- a/test/Driver/openmp-offload.c
+++ b/test/Driver/openmp-offload.c
@@ -106,15 +106,18 @@
 // CHK-PHASES: 2: compiler, {1}, ir, (host-openmp)
 // CHK-PHASES: 3: backend, {2}, assembler, (host-openmp)
 // CHK-PHASES: 4: assembler, {3}, object, (host-openmp)
-// CHK-PHASES: 5: linker, {4}, image, (host-openmp)
-// CHK-PHASES: 6: input, "[[INPUT]]", c, (device-openmp)
-// CHK-PHASES: 7: preprocessor, {6}, cpp-output, (device-openmp)
-// CHK-PHASES: 8: compiler, {7}, ir, (device-openmp)
-// CHK-PHASES: 9: offload, "host-openmp (powerpc64le-ibm-linux-gnu)" {2}, "device-openmp (x86_64-pc-linux-gnu)" {8}, ir
-// CHK-PHASES: 10: backend, {9}, assembler, (device-openmp)
-// CHK-PHASES: 11: assembler, {10}, object, (device-openmp)
-// CHK-PHASES: 12: linker, {11}, image, (device-openmp)
-// CHK-PHASES: 13: offload, "host-openmp (powerpc64le-ibm-linux-gnu)" {5}, "device-openmp (x86_64-pc-linux-gnu)" {12}, image
+// CHK-PHASES: 5: input, "[[INPUT]]", c, (device-openmp)
+// CHK-PHASES: 6: preprocessor, {5}, cpp-output, (device-openmp)
+// CHK-PHASES: 7: compiler, {6}, ir, (device-openmp)
+// CHK-PHASES: 8: offload, "host-openmp (powerpc64le-ibm-linux-gnu)" {2}, "device-openmp (x86_64-pc-linux-gnu)" {7}, ir
+// CHK-PHASES: 9: backend, {8}, assembler, (device-openmp)
+// CHK-PHASES: 10: assembler, {9}, object, (device-openmp)
+// CHK-PHASES: 11: linker, {10}, image, (device-openmp)
+// CHK-PHASES: 12: offload, "device-openmp (x86_64-pc-linux-gnu)" {11}, image
+// CHK-PHASES: 13: clang-offload-wrapper, {12}, ir, (host-openmp)
+// CHK-PHASES: 14: backend, {13}, assembler, (host-openmp)
+// CHK-PHASES: 15: assembler, {14}, object, (host-openmp)
+// CHK-PHASES: 16: linker, {4, 15}, image, (host-openmp)
 
 /// ###########################################################################
 
@@ -128,15 +131,15 @@
 // CHK-PHASES-LIB: 3: compiler, {2}, ir, (host-openmp)
 // CHK-PHASES-LIB: 4: backend, {3}, assembler, (host-openmp)
 // CHK-PHASES-LIB: 5: assembler, {4}, object, (host-openmp)
-// CHK-PHASES-LIB: 6: linker, {0, 5}, image, (host-openmp)
-// CHK-PHASES-LIB: 7: input, "somelib", object, (device-openmp)
-// CHK-PHASES-LIB: 8: input, "[[INPUT]]", c, (device-openmp)
-// CHK-PHASES-LIB: 9: preprocessor, {8}, cpp-output, (device-openmp)
-// CHK-PHASES-LIB: 10: compiler, {9}, ir, (device-openmp)
-// CHK-PHASES-LIB: 11: offload, "host-openmp (powerpc64-ibm-linux-gnu)" {3}, "device-openmp (x86_64-pc-linux-gnu)" {10}, ir
-// CHK-PHASES-LIB: 12: backend, {11}, assembler, (device-openmp)
-// CHK-PHASES-LIB: 13: assembler, {12}, object, (device-openmp)
-// CHK-PHASES-LIB: 14: linker, {7, 13}, image, (device-openmp)
+// CHK-PHASES-LIB: 6: input, "somelib", object, (device-openmp)
+// CHK-PHASES-LIB: 7: input, "[[INPUT]]", c, (device-openmp)
+// CHK-PHASES-LIB: 8: preprocessor, {7}, cpp-output, (device-openmp)
+// CHK-PHASES-LIB: 9: compiler, {8}, ir, (device-openmp)
+// CHK-PHASES-LIB: 10: offload, "host-openmp (powerpc64-ibm-linux-gnu)" {3}, "device-openmp (x86_64-pc-linux-gnu)" {9}, ir
+// CHK-PHASES-LIB: 11: backend, {10}, assembler, (device-openmp)
+// CHK-PHASES-LIB: 12: assembler, {11}, object, (device-openmp)
+// CHK-PHASES-LIB: 13: linker, {6, 12}, image, (device-openmp)
+// CHK-PHASES-LIB: 14: offload, "device-openmp (x86_64-pc-linux-gnu)" {13}, image
 // CHK-PHASES-LIB: 15: input, "somelib", object, (device-openmp)
 // CHK-PHASES-LIB: 16: input, "[[INPUT]]", c, (device-openmp)
 // CHK-PHASES-LIB: 17: preprocessor, {16}, cpp-output, (device-openmp)
@@ -145,8 +148,11 @@
 // CHK-PHASES-LIB: 20: backend, {19}, assembler, (device-openmp)
 // CHK-PHASES-LIB: 21: assembler, {20}, object, (device-openmp)
 // CHK-PHASES-LIB: 22: linker, {15, 21}, image, (device-openmp)
-// CHK-PHASES-LIB: 23: offload, "host-openmp (powerpc64-ibm-linux-gnu)" {6}, "device-openmp (x86_64-pc-linux-gnu)" {14}, "device-openmp (powerpc64-ibm-linux-gnu)" {22}, image
-
+// CHK-PHASES-LIB: 23: offload, "device-openmp (powerpc64-ibm-linux-gnu)" {22}, image
+// CHK-PHASES-LIB: 24: clang-offload-wrapper, {14, 23}, ir, (host-openmp)
+// CHK-PHASES-LIB: 25: backend, {24}, assembler, (host-openmp)
+// CHK-PHASES-LIB: 26: assembler, {25}, object, (host-openmp)
+// CHK-PHASES-LIB: 27: linker, {0, 5, 26}, image, (host-openmp)
 
 /// ###########################################################################
 
@@ -165,21 +171,21 @@
 // CHK-PHASES-FILES: 8: compiler, {7}, ir, (host-openmp)
 // CHK-PHASES-FILES: 9: backend, {8}, assembler, (host-openmp)
 // CHK-PHASES-FILES: 10: assembler, {9}, object, (host-openmp)
-// CHK-PHASES-FILES: 11: linker, {0, 5, 10}, image, (host-openmp)
-// CHK-PHASES-FILES: 12: input, "somelib", object, (device-openmp)
-// CHK-PHASES-FILES: 13: input, "[[INPUT1]]", c, (device-openmp)
-// CHK-PHASES-FILES: 14: preprocessor, {13}, cpp-output, (device-openmp)
-// CHK-PHASES-FILES: 15: compiler, {14}, ir, (device-openmp)
-// CHK-PHASES-FILES: 16: offload, "host-openmp (powerpc64-ibm-linux-gnu)" {3}, "device-openmp (x86_64-pc-linux-gnu)" {15}, ir
-// CHK-PHASES-FILES: 17: backend, {16}, assembler, (device-openmp)
-// CHK-PHASES-FILES: 18: assembler, {17}, object, (device-openmp)
-// CHK-PHASES-FILES: 19: input, "[[INPUT2]]", c, (device-openmp)
-// CHK-PHASES-FILES: 20: preprocessor, {19}, cpp-output, (device-openmp)
-// CHK-PHASES-FILES: 21: compiler, {20}, ir, (device-openmp)
-// CHK-PHASES-FILES: 22: offload, "host-openmp (powerpc64-ibm-linux-gnu)" {8}, "device-openmp (x86_64-pc-linux-gnu)" {21}, ir
-// CHK-PHASES-FILES: 23: backend, {22}, assembler, (device-openmp)
-// CHK-PHASES-FILES: 24: assembler, {23}, object, (device-openmp)
-// CHK-PHASES-FILES: 25: linker, {12, 18, 24}, image, (device-openmp)
+// CHK-PHASES-FILES: 11: input, "somelib", object, (device-openmp)
+// CHK-PHASES-FILES: 12: input, "[[INPUT1]]", c, (device-openmp)
+// CHK-PHASES-FILES: 13: preprocessor, {12}, cpp-output, (device-openmp)
+// CHK-PHASES-FILES: 14: compiler, {13}, ir, (device-openmp)
+// CHK-PHASES-FILES: 15: offload, "host-openmp (powerpc64-ibm-linux-gnu)" {3}, "device-openmp (x86_64-pc-linux-gnu)" {14}, ir
+// CHK-PHASES-FILES: 16: backend, {15}, assembler, (device-openmp)
+// CHK-PHASES-FILES: 17: assembler, {16}, object, (device-openmp)
+// CHK-PHASES-FILES: 18: input, "[[INPUT2]]", c, (device-openmp)
+// CHK-PHASES-FILES: 19: preprocessor, {18}, cpp-output, (device-openmp)
+// CHK-PHASES-FILES: 20: compiler, {19}, ir, (device-openmp)
+// CHK-PHASES-FILES: 21: offload, "host-openmp (powerpc64-ibm-linux-gnu)" {8}, "device-openmp (x86_64-pc-linux-gnu)" {20}, ir
+// CHK-PHASES-FILES: 22: backend, {21}, assembler, (device-openmp)
+// CHK-PHASES-FILES: 23: assembler, {22}, object, (device-openmp)
+// CHK-PHASES-FILES: 24: linker, {11, 17, 23}, image, (device-openmp)
+// CHK-PHASES-FILES: 25: offload, "device-openmp (x86_64-pc-linux-gnu)" {24}, image
 // CHK-PHASES-FILES: 26: input, "somelib", object, (device-openmp)
 // CHK-PHASES-FILES: 27: input, "[[INPUT1]]", c, (device-openmp)
 // CHK-PHASES-FILES: 28: preprocessor, {27}, cpp-output, (device-openmp)
@@ -194,7 +200,11 @@
 // CHK-PHASES-FILES: 37: backend, {36}, assembler, (device-openmp)
 // CHK-PHASES-FILES: 38: assembler, {37}, object, (device-openmp)
 // CHK-PHASES-FILES: 39: linker, {26, 32, 38}, image, (device-openmp)
-// CHK-PHASES-FILES: 40: offload, "host-openmp (powerpc64-ibm-linux-gnu)" {11}, "device-openmp (x86_64-pc-linux-gnu)" {25}, "device-openmp (powerpc64-ibm-linux-gnu)" {39}, image
+// CHK-PHASES-FILES: 40: offload, "device-openmp (powerpc64-ibm-linux-gnu)" {39}, image
+// CHK-PHASES-FILES: 41: clang-offload-wrapper, {25, 40}, ir, (host-openmp)
+// CHK-PHASES-FILES: 42: backend, {41}, assembler, (host-openmp)
+// CHK-PHASES-FILES: 43: assembler, {42}, object, (host-openmp)
+// CHK-PHASES-FILES: 44: linker, {0, 5, 10, 43}, image, (host-openmp)
 
 /// ###########################################################################
 
@@ -216,15 +226,18 @@
 // CHK-PHASES-WITH-CUDA: 11: offload, "host-cuda-openmp (powerpc64le-ibm-linux-gnu)" {2}, "device-cuda (nvptx64-nvidia-cuda)" {10}, ir
 // CHK-PHASES-WITH-CUDA: 12: backend, {11}, assembler, (host-cuda-openmp)
 // CHK-PHASES-WITH-CUDA: 13: assembler, {12}, object, (host-cuda-openmp)
-// CHK-PHASES-WITH-CUDA: 14: linker, {13}, image, (host-cuda-openmp)
-// CHK-PHASES-WITH-CUDA: 15: input, "[[INPUT]]", cuda, (device-openmp)
-// CHK-PHASES-WITH-CUDA: 16: preprocessor, {15}, cuda-cpp-output, (device-openmp)
-// CHK-PHASES-WITH-CUDA: 17: compiler, {16}, ir, (device-openmp)
-// CHK-PHASES-WITH-CUDA: 18: offload, "host-cuda-openmp (powerpc64le-ibm-linux-gnu)" {2}, "device-openmp (nvptx64-nvidia-cuda)" {17}, ir
-// CHK-PHASES-WITH-CUDA: 19: backend, {18}, assembler, (device-openmp)
-// CHK-PHASES-WITH-CUDA: 20: assembler, {19}, object, (device-openmp)
-// CHK-PHASES-WITH-CUDA: 21: linker, {20}, image, (device-openmp)
-// CHK-PHASES-WITH-CUDA: 22: offload, "host-cuda-openmp (powerpc64le-ibm-linux-gnu)" {14}, "device-openmp (nvptx64-nvidia-cuda)" {21}, image
+// CHK-PHASES-WITH-CUDA: 14: input, "[[INPUT]]", cuda, (device-openmp)
+// CHK-PHASES-WITH-CUDA: 15: preprocessor, {14}, cuda-cpp-output, (device-openmp)
+// CHK-PHASES-WITH-CUDA: 16: compiler, {15}, ir, (device-openmp)
+// CHK-PHASES-WITH-CUDA: 17: offload, "host-cuda-openmp (powerpc64le-ibm-linux-gnu)" {2}, "device-openmp (nvptx64-nvidia-cuda)" {16}, ir
+// CHK-PHASES-WITH-CUDA: 18: backend, {17}, assembler, (device-openmp)
+// CHK-PHASES-WITH-CUDA: 19: assembler, {18}, object, (device-openmp)
+// CHK-PHASES-WITH-CUDA: 20: linker, {19}, image, (device-openmp)
+// CHK-PHASES-WITH-CUDA: 21: offload, "device-openmp (nvptx64-nvidia-cuda)" {20}, image
+// CHK-PHASES-WITH-CUDA: 22: clang-offload-wrapper, {21}, ir, (host-cuda-openmp)
+// CHK-PHASES-WITH-CUDA: 23: backend, {22}, assembler, (host-cuda-openmp)
+// CHK-PHASES-WITH-CUDA: 24: assembler, {23}, object, (host-cuda-openmp)
+// CHK-PHASES-WITH-CUDA: 25: linker, {13, 24}, image, (host-cuda-openmp)
 
 /// ###########################################################################
 
@@ -237,65 +250,31 @@
 /// -fopenmp-host-ir-file-path: specifies the host IR file that can be loaded by
 /// the target code generation to gather information about which declaration
 /// really need to be emitted.
-/// We use -fopenmp-dump-offload-linker-script to dump the linker script and
-/// check its contents.
 ///
-// RUN:   %clang -### -fopenmp=libomp -o %t.out -target powerpc64le-linux -fopenmp-targets=powerpc64le-ibm-linux-gnu,x86_64-pc-linux-gnu %s -fopenmp-dump-offload-linker-script -no-canonical-prefixes 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHK-COMMANDS -check-prefix=CHK-LKS -check-prefix=CHK-LKS-REG %s
-// RUN:   %clang -### -fopenmp=libomp -o %t.out -target powerpc64le-linux -fopenmp-targets=powerpc64le-ibm-linux-gnu,x86_64-pc-linux-gnu %s -save-temps -fopenmp-dump-offload-linker-script -no-canonical-prefixes 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHK-COMMANDS-ST -check-prefix=CHK-LKS -check-prefix=CHK-LKS-ST %s
-
-// Make sure we are not dumping the script unless the user requested it.
 // RUN:   %clang -### -fopenmp=libomp -o %t.out -target powerpc64le-linux -fopenmp-targets=powerpc64le-ibm-linux-gnu,x86_64-pc-linux-gnu %s -no-canonical-prefixes 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHK-LKS-NODUMP %s
+// RUN:   | FileCheck -check-prefix=CHK-COMMANDS %s
 // RUN:   %clang -### -fopenmp=libomp -o %t.out -target powerpc64le-linux -fopenmp-targets=powerpc64le-ibm-linux-gnu,x86_64-pc-linux-gnu %s -save-temps -no-canonical-prefixes 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHK-LKS-NODUMP %s
-
-//
-// Check the linker script contains what we expect.
-//
-// CHK-LKS: /*
-// CHK-LKS:                   OpenMP Offload Linker Script
-// CHK-LKS:             *** Automatically generated by Clang ***
-// CHK-LKS-NODUMP-NOT:  OpenMP Offload Linker Script.
-// CHK-LKS: */
-// CHK-LKS: TARGET(binary)
-// CHK-LKS-REG: INPUT([[T1BIN:.+\.out]])
-// CHK-LKS-REG: INPUT([[T2BIN:.+\.out]])
-// CHK-LKS-ST: INPUT([[T1BIN:.+\.out-openmp-powerpc64le-ibm-linux-gnu]])
-// CHK-LKS-ST: INPUT([[T2BIN:.+\.out-openmp-x86_64-pc-linux-gnu]])
-// CHK-LKS: SECTIONS
-// CHK-LKS: {
-// CHK-LKS:   .omp_offloading.powerpc64le-ibm-linux-gnu :
-// CHK-LKS:   ALIGN(0x10)
-// CHK-LKS:   {
-// CHK-LKS:     PROVIDE_HIDDEN(.omp_offloading.img_start.powerpc64le-ibm-linux-gnu = .);
-// CHK-LKS:     [[T1BIN]]
-// CHK-LKS:     PROVIDE_HIDDEN(.omp_offloading.img_end.powerpc64le-ibm-linux-gnu = .);
-// CHK-LKS:   }
-// CHK-LKS:   .omp_offloading.x86_64-pc-linux-gnu :
-// CHK-LKS:   ALIGN(0x10)
-// CHK-LKS:   {
-// CHK-LKS:     PROVIDE_HIDDEN(.omp_offloading.img_start.x86_64-pc-linux-gnu = .);
-// CHK-LKS:     [[T2BIN]]
-// CHK-LKS:     PROVIDE_HIDDEN(.omp_offloading.img_end.x86_64-pc-linux-gnu = .);
-// CHK-LKS:   }
-// CHK-LKS: }
-// CHK-LKS: INSERT BEFORE .data
+// RUN:   | FileCheck -check-prefix=CHK-COMMANDS-ST %s
 
 //
-// Generate host BC file.
+// Generate host BC file and host object.
 //
 // CHK-COMMANDS: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-emit-llvm-bc"
 // CHK-COMMANDS-SAME: "-fopenmp-targets=powerpc64le-ibm-linux-gnu,x86_64-pc-linux-gnu"
 // CHK-COMMANDS-SAME: "-o" "
 // CHK-COMMANDS-SAME: [[HOSTBC:[^\\/]+\.bc]]" "-x" "c" "
 // CHK-COMMANDS-SAME: [[INPUT:[^\\/]+\.c]]"
+// CHK-COMMANDS: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-emit-obj" {{.*}}"-fopenmp" {{.*}}"-o" "
+// CHK-COMMANDS-SAME: [[HOSTOBJ:[^\\/]+\.o]]" "-x" "ir" "{{.*}}[[HOSTBC]]"
 // CHK-COMMANDS-ST: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-E" {{.*}}"-fopenmp" {{.*}}"-o" "
 // CHK-COMMANDS-ST-SAME: [[HOSTPP:[^\\/]+\.i]]" "-x" "c" "
 // CHK-COMMANDS-ST-SAME: [[INPUT:[^\\/]+\.c]]"
 // CHK-COMMANDS-ST: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-emit-llvm-bc" {{.*}}"-fopenmp" {{.*}}"-fopenmp-targets=powerpc64le-ibm-linux-gnu,x86_64-pc-linux-gnu" {{.*}}"-o" "
 // CHK-COMMANDS-ST-SAME: [[HOSTBC:[^\\/]+\.bc]]" "-x" "cpp-output" "{{.*}}[[HOSTPP]]"
+// CHK-COMMANDS-ST: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-S" {{.*}}"-fopenmp" {{.*}}"-o" "
+// CHK-COMMANDS-ST-SAME: [[HOSTASM:[^\\/]+\.s]]" "-x" "ir" "{{.*}}[[HOSTBC]]"
+// CHK-COMMANDS-ST: clang{{.*}}" "-cc1as" "-triple" "powerpc64le-unknown-linux" "-filetype" "obj" {{.*}}"-o" "
+// CHK-COMMANDS-ST-SAME: [[HOSTOBJ:[^\\/]+\.o]]" "{{.*}}[[HOSTASM]]"
 
 //
 // Compile for the powerpc device.
@@ -335,21 +314,26 @@
 // CHK-COMMANDS-ST-SAME: [[T2BIN:[^\\/]+\.out-openmp-x86_64-pc-linux-gnu]]" {{.*}}"{{.*}}[[T2OBJ]]"
 
 //
-// Generate host object from the BC file and link using the linker script.
+// Create wrapper BC file and wrapper object.
 //
+// CHK-COMMANDS: clang-offload-wrapper{{(\.exe)?}}" "-target" "powerpc64le-unknown-linux" {{.*}}"-o" "
+// CHK-COMMANDS-SAME: [[WRAPPERBC:[^\\/]+\.bc]]" "{{.*}}[[T1BIN]]" "{{.*}}[[T2BIN]]"
 // CHK-COMMANDS: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-emit-obj" {{.*}}"-fopenmp" {{.*}}"-o" "
-// CHK-COMMANDS-SAME: [[HOSTOBJ:[^\\/]+\.o]]" "-x" "ir" "{{.*}}[[HOSTBC]]"
-// CHK-COMMANDS: ld{{(\.exe)?}}" {{.*}}"-o" "
-// CHK-COMMANDS-SAME: [[HOSTBIN:[^\\/]+\.out]]"  {{.*}}"-lomptarget" {{.*}}"-T" "
-// CHK-COMMANDS-SAME: [[HOSTLK:[^\\/]+\.lk]]"
+// CHK-COMMANDS-SAME: [[WRAPPEROBJ:[^\\/]+\.o]]" "-x" "ir" "{{.*}}[[WRAPPERBC]]"
+// CHK-COMMANDS-ST: clang-offload-wrapper{{(\.exe)?}}" "-target" "powerpc64le-unknown-linux" {{.*}}"-o" "
+// CHK-COMMANDS-ST-SAME: [[WRAPPERBC:[^\\/]+\.bc]]" "{{.*}}[[T1BIN]]" "{{.*}}[[T2BIN]]"
 // CHK-COMMANDS-ST: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-S" {{.*}}"-fopenmp" {{.*}}"-o" "
-// CHK-COMMANDS-ST-SAME: [[HOSTASM:[^\\/]+\.s]]" "-x" "ir" "{{.*}}[[HOSTBC]]"
+// CHK-COMMANDS-ST-SAME: [[WRAPPERASM:[^\\/]+\.s]]" "-x" "ir" "{{.*}}[[WRAPPERBC]]"
 // CHK-COMMANDS-ST: clang{{.*}}" "-cc1as" "-triple" "powerpc64le-unknown-linux" "-filetype" "obj" {{.*}}"-o" "
-// CHK-COMMANDS-ST-SAME: [[HOSTOBJ:[^\\/]+\.o]]" "{{.*}}[[HOSTASM]]"
-// CHK-COMMANDS-ST: ld{{(\.exe)?}}" {{.*}}"-o" "
-// CHK-COMMANDS-ST-SAME: [[HOSTBIN:[^\\/]+\.out]]"  {{.*}}"-lomptarget" {{.*}}"-T" "
-// CHK-COMMANDS-ST-SAME: [[HOSTLK:[^\\/]+\.lk]]"
+// CHK-COMMANDS-ST-SAME: [[WRAPPEROBJ:[^\\/]+\.o]]" "{{.*}}[[WRAPPERASM]]"
 
+//
+// Link host binary.
+//
+// CHK-COMMANDS: ld{{(\.exe)?}}" {{.*}}"-o" "
+// CHK-COMMANDS-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" "{{.*}}[[WRAPPEROBJ]]" {{.*}}"-lomptarget"
+// CHK-COMMANDS-ST: ld{{(\.exe)?}}" {{.*}}"-o" "
+// CHK-COMMANDS-ST-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" "{{.*}}[[WRAPPEROBJ]]" {{.*}}"-lomptarget"
 
 /// ###########################################################################
 
@@ -391,20 +375,24 @@
 // CHK-UBACTIONS: 3: compiler, {2}, ir, (host-openmp)
 // CHK-UBACTIONS: 4: backend, {3}, assembler, (host-openmp)
 // CHK-UBACTIONS: 5: assembler, {4}, object, (host-openmp)
-// CHK-UBACTIONS: 6: linker, {0, 5}, image, (host-openmp)
-// CHK-UBACTIONS: 7: input, "somelib", object, (device-openmp)
-// CHK-UBACTIONS: 8: compiler, {2}, ir, (device-openmp)
-// CHK-UBACTIONS: 9: offload, "host-openmp (powerpc64le-unknown-linux)" {3}, "device-openmp (powerpc64le-ibm-linux-gnu)" {8}, ir
-// CHK-UBACTIONS: 10: backend, {9}, assembler, (device-openmp)
-// CHK-UBACTIONS: 11: assembler, {10}, object, (device-openmp)
-// CHK-UBACTIONS: 12: linker, {7, 11}, image, (device-openmp)
+// CHK-UBACTIONS: 6: input, "somelib", object, (device-openmp)
+// CHK-UBACTIONS: 7: compiler, {2}, ir, (device-openmp)
+// CHK-UBACTIONS: 8: offload, "host-openmp (powerpc64le-unknown-linux)" {3}, "device-openmp (powerpc64le-ibm-linux-gnu)" {7}, ir
+// CHK-UBACTIONS: 9: backend, {8}, assembler, (device-openmp)
+// CHK-UBACTIONS: 10: assembler, {9}, object, (device-openmp)
+// CHK-UBACTIONS: 11: linker, {6, 10}, image, (device-openmp)
+// CHK-UBACTIONS: 12: offload, "device-openmp (powerpc64le-ibm-linux-gnu)" {11}, image
 // CHK-UBACTIONS: 13: input, "somelib", object, (device-openmp)
 // CHK-UBACTIONS: 14: compiler, {2}, ir, (device-openmp)
 // CHK-UBACTIONS: 15: offload, "host-openmp (powerpc64le-unknown-linux)" {3}, "device-openmp (x86_64-pc-linux-gnu)" {14}, ir
 // CHK-UBACTIONS: 16: backend, {15}, assembler, (device-openmp)
 // CHK-UBACTIONS: 17: assembler, {16}, object, (device-openmp)
 // CHK-UBACTIONS: 18: linker, {13, 17}, image, (device-openmp)
-// CHK-UBACTIONS: 19: offload, "host-openmp (powerpc64le-unknown-linux)" {6}, "device-openmp (powerpc64le-ibm-linux-gnu)" {12}, "device-openmp (x86_64-pc-linux-gnu)" {18}, image
+// CHK-UBACTIONS: 19: offload, "device-openmp (x86_64-pc-linux-gnu)" {18}, image
+// CHK-UBACTIONS: 20: clang-offload-wrapper, {12, 19}, ir, (host-openmp)
+// CHK-UBACTIONS: 21: backend, {20}, assembler, (host-openmp)
+// CHK-UBACTIONS: 22: assembler, {21}, object, (host-openmp)
+// CHK-UBACTIONS: 23: linker, {0, 5, 22}, image, (host-openmp)
 
 /// ###########################################################################
 
@@ -507,6 +495,8 @@
 // CHK-UBJOBS-SAME: [[T2PP:[^\\/]+\.i]]" "-unbundle"
 // CHK-UBJOBS: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-emit-llvm-bc"  {{.*}}"-fopenmp" {{.*}}"-fopenmp-targets=powerpc64le-ibm-linux-gnu,x86_64-pc-linux-gnu" {{.*}}"-o" "
 // CHK-UBJOBS-SAME: [[HOSTBC:[^\\/]+\.bc]]" "-x" "cpp-output" "{{.*}}[[HOSTPP]]"
+// CHK-UBJOBS: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-emit-obj" {{.*}}"-fopenmp" {{.*}}"-o" "
+// CHK-UBJOBS-SAME: [[HOSTOBJ:[^\\/]+\.o]]" "-x" "ir" "{{.*}}[[HOSTBC]]"
 // CHK-UBJOBS-ST: clang-offload-bundler{{.*}}" "-type=i" "-targets=host-powerpc64le-unknown-linux,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu" "-inputs=
 // CHK-UBJOBS-ST-SAME: [[INPUT:[^\\/]+\.i]]" "-outputs=
 // CHK-UBJOBS-ST-SAME: [[HOSTPP:[^\\/,]+\.i]],
@@ -514,6 +504,10 @@
 // CHK-UBJOBS-ST-SAME: [[T2PP:[^\\/,]+\.i]]" "-unbundle"
 // CHK-UBJOBS-ST: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-emit-llvm-bc"  {{.*}}"-fopenmp" {{.*}}"-fopenmp-targets=powerpc64le-ibm-linux-gnu,x86_64-pc-linux-gnu" {{.*}}"-o" "
 // CHK-UBJOBS-ST-SAME: [[HOSTBC:[^\\/]+\.bc]]" "-x" "cpp-output" "{{.*}}[[HOSTPP]]"
+// CHK-UBJOBS-ST: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-S" {{.*}}"-fopenmp" {{.*}}"-o" "
+// CHK-UBJOBS-ST-SAME: [[HOSTASM:[^\\/]+\.s]]" "-x" "ir" "{{.*}}[[HOSTBC]]"
+// CHK-UBJOBS-ST: clang{{.*}}" "-cc1as" "-triple" "powerpc64le-unknown-linux" "-filetype" "obj" {{.*}}"-o" "
+// CHK-UBJOBS-ST-SAME: [[HOSTOBJ:[^\\/]+\.o]]" "{{.*}}[[HOSTASM]]"
 
 // Create target 1 object.
 // CHK-UBJOBS: clang{{.*}}" "-cc1" "-triple" "powerpc64le-ibm-linux-gnu" "-aux-triple" "powerpc64le-unknown-linux" {{.*}}"-emit-obj" {{.*}}"-fopenmp" {{.*}}"-fopenmp-is-device" "-fopenmp-host-ir-file-path" "{{.*}}[[HOSTBC]]" {{.*}}"-o" "
@@ -543,19 +537,23 @@
 // CHK-UBJOBS-ST: ld{{(\.exe)?}}" {{.*}}"-o" "
 // CHK-UBJOBS-ST-SAME: [[T2BIN:[^\\/]+\.out-openmp-x86_64-pc-linux-gnu]]" {{.*}}"{{.*}}[[T2OBJ]]"
 
-// Create binary.
+// Create wrapper BC file and wrapper object.
+// CHK-UBJOBS: clang-offload-wrapper{{(\.exe)?}}" "-target" "powerpc64le-unknown-linux" {{.*}}"-o" "
+// CHK-UBJOBS-SAME: [[WRAPPERBC:[^\\/]+\.bc]]" "{{.*}}[[T1BIN]]" "{{.*}}[[T2BIN]]"
 // CHK-UBJOBS: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-emit-obj" {{.*}}"-fopenmp" {{.*}}"-o" "
-// CHK-UBJOBS-SAME: [[HOSTOBJ:[^\\/]+\.o]]" "-x" "ir" "{{.*}}[[HOSTBC]]"
-// CHK-UBJOBS: ld{{(\.exe)?}}" {{.*}}"-o" "
-// CHK-UBJOBS-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" {{.*}}"-T" "
-// CHK-UBJOBS-SAME: [[LKS:[^\\/]+\.lk]]"
+// CHK-UBJOBS-SAME: [[WRAPPEROBJ:[^\\/]+\.o]]" "-x" "ir" "{{.*}}[[WRAPPERBC]]"
+// CHK-UBJOBS-ST: clang-offload-wrapper{{(\.exe)?}}" "-target" "powerpc64le-unknown-linux" {{.*}}"-o" "
+// CHK-UBJOBS-ST-SAME: [[WRAPPERBC:[^\\/]+\.bc]]" "{{.*}}[[T1BIN]]" "{{.*}}[[T2BIN]]"
 // CHK-UBJOBS-ST: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" {{.*}}"-S" {{.*}}"-fopenmp" {{.*}}"-o" "
-// CHK-UBJOBS-ST-SAME: [[HOSTASM:[^\\/]+\.s]]" "-x" "ir" "{{.*}}[[HOSTBC]]"
+// CHK-UBJOBS-ST-SAME: [[WRAPPERASM:[^\\/]+\.s]]" "-x" "ir" "{{.*}}[[WRAPPERBC]]"
 // CHK-UBJOBS-ST: clang{{.*}}" "-cc1as" "-triple" "powerpc64le-unknown-linux" "-filetype" "obj" {{.*}}"-o" "
-// CHK-UBJOBS-ST-SAME: [[HOSTOBJ:[^\\/]+\.o]]" "{{.*}}[[HOSTASM]]"
+// CHK-UBJOBS-ST-SAME: [[WRAPPEROBJ:[^\\/]+\.o]]" "{{.*}}[[WRAPPERASM]]"
+
+// Create binary.
+// CHK-UBJOBS: ld{{(\.exe)?}}" {{.*}}"-o" "
+// CHK-UBJOBS-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" "{{.*}}[[WRAPPEROBJ]]"
 // CHK-UBJOBS-ST: ld{{(\.exe)?}}" {{.*}}"-o" "
-// CHK-UBJOBS-ST-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" {{.*}}"-T" "
-// CHK-UBJOBS-ST-SAME: [[LKS:[^\\/]+\.lk]]"
+// CHK-UBJOBS-ST-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" "{{.*}}[[WRAPPEROBJ]]"
 
 // Unbundle object file.
 // CHK-UBJOBS2: clang-offload-bundler{{.*}}" "-type=o" "-targets=host-powerpc64le-unknown-linux,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu" "-inputs=
@@ -567,9 +565,12 @@
 // CHK-UBJOBS2-SAME: [[T1BIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[T1OBJ]]"
 // CHK-UBJOBS2: ld{{(\.exe)?}}" {{.*}}"-o" "
 // CHK-UBJOBS2-SAME: [[T2BIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[T2OBJ]]"
+// CHK-UBJOBS2: clang-offload-wrapper{{(\.exe)?}}" "-target" "powerpc64le-unknown-linux" {{.*}}"-o" "
+// CHK-UBJOBS2-SAME: [[WRAPPERBC:[^\\/]+\.bc]]" "{{.*}}[[T1BIN]]" "{{.*}}[[T2BIN]]"
+// CHK-UBJOBS2: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" "-emit-obj" {{.*}}"-fopenmp" {{.*}}"-o" "
+// CHK-UBJOBS2-SAME: [[WRAPPEROBJ:[^\\/]+\.o]]" "-x" "ir" "{{.*}}[[WRAPPERBC]]"
 // CHK-UBJOBS2: ld{{(\.exe)?}}" {{.*}}"-o" "
-// CHK-UBJOBS2-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" {{.*}}"-T" "
-// CHK-UBJOBS2-SAME: [[LKS:[^\\/]+\.lk]]"
+// CHK-UBJOBS2-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" "{{.*}}[[WRAPPEROBJ]]"
 // CHK-UBJOBS2-ST-NOT: clang-offload-bundler{{.*}}in.so
 // CHK-UBJOBS2-ST: clang-offload-bundler{{.*}}" "-type=o" "-targets=host-powerpc64le-unknown-linux,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu" "-inputs=
 // CHK-UBJOBS2-ST-SAME: [[INPUT:[^\\/]+\.o]]" "-outputs=
@@ -581,9 +582,14 @@
 // CHK-UBJOBS2-ST-SAME: [[T1BIN:[^\\/]+\.out-openmp-powerpc64le-ibm-linux-gnu]]" {{.*}}"{{.*}}[[T1OBJ]]"
 // CHK-UBJOBS2-ST: ld{{(\.exe)?}}" {{.*}}"-o" "
 // CHK-UBJOBS2-ST-SAME: [[T2BIN:[^\\/]+\.out-openmp-x86_64-pc-linux-gnu]]" {{.*}}"{{.*}}[[T2OBJ]]"
+// CHK-UBJOBS2-ST: clang-offload-wrapper{{(\.exe)?}}" "-target" "powerpc64le-unknown-linux" {{.*}}"-o" "
+// CHK-UBJOBS2-ST-SAME: [[WRAPPERBC:[^\\/]+\.bc]]" "{{.*}}[[T1BIN]]" "{{.*}}[[T2BIN]]"
+// CHK-UBJOBS2-ST: clang{{.*}}" "-cc1" "-triple" "powerpc64le-unknown-linux" "-S" {{.*}}"-fopenmp" {{.*}}"-o" "
+// CHK-UBJOBS2-ST-SAME: [[WRAPPERASM:[^\\/]+\.s]]" "-x" "ir" "{{.*}}[[WRAPPERBC]]"
+// CHK-UBJOBS2-ST: clang{{.*}}" "-cc1as" "-triple" "powerpc64le-unknown-linux" "-filetype" "obj" {{.*}}"-o" "
+// CHK-UBJOBS2-ST-SAME: [[WRAPPEROBJ:[^\\/]+\.o]]" "{{.*}}[[WRAPPERASM]]"
 // CHK-UBJOBS2-ST: ld{{(\.exe)?}}" {{.*}}"-o" "
-// CHK-UBJOBS2-ST-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" {{.*}}"-T" "
-// CHK-UBJOBS2-ST-SAME: [[LKS:[^\\/]+\.lk]]"
+// CHK-UBJOBS2-ST-SAME: [[HOSTBIN:[^\\/]+\.out]]" {{.*}}"{{.*}}[[HOSTOBJ]]" "{{.*}}[[WRAPPEROBJ]]"
 
 /// ###########################################################################