Prune unreachable targets when switching on an enum discriminant ✨nox/i-need-a-niche-but-i-have-no-dog

Need to write tests, but it's Sunday evening and I have some chores to do.

6 files changed, 651 insertions, 2 deletions

diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index ec8f8edfca0..a76cbc771b5 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -2,6 +2,7 @@
 #![feature(associated_type_bounds)]
 #![feature(bool_to_option)]
 #![feature(box_patterns)]
+#![feature(exact_size_is_empty)]
 #![feature(in_band_lifetimes)]
 #![feature(let_else)]
 #![feature(nll)]
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 96dc3394cd8..0b8e4a37432 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -249,7 +249,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // `switch_ty` is redundant, sanity-check that.
         assert_eq!(discr.layout.ty, switch_ty);
         let mut target_iter = targets.iter();
-        if target_iter.len() == 1 {
+        if target_iter.is_empty() {
+            helper.funclet_br(self, &mut bx, targets.otherwise());
+        } else if target_iter.len() == 1 {
             // If there are two targets (one conditional, one fallback), emit br instead of switch
             let (test_value, target) = target_iter.next().unwrap();
             let lltrue = helper.llblock(self, target);
@@ -992,7 +994,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let mir = self.mir;
         let data = &mir[bb];
         let statements = &data.statements;
-        let terminator = data.terminator();
+        let mut terminator = data.terminator();
 
         debug!("codegen_block({:?}={:?})", bb, data);
 
@@ -1000,9 +1002,145 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             bx = self.codegen_statement(bx, statement);
         }
 
+        let optimized_terminator;
+        if let Some((enum_ty, discr, switch_ty, targets)) =
+            self.get_discriminant_switch_block_parts(statements, terminator)
+        {
+            let _span = debug_span!(
+                "opt_switch_monomorphizing",
+                "type" = %enum_ty,
+                terminator = ?terminator.kind,
+                span = ?terminator.source_info.span,
+            )
+            .entered();
+
+            let enum_layout = bx.cx().layout_of(enum_ty);
+            let tcx = bx.cx().tcx();
+            let discr_for_variant = |index| {
+                enum_layout
+                    .ty
+                    .discriminant_for_variant(tcx, index)
+                    .map_or(index.as_u32() as u128, |discr| discr.val)
+            };
+
+            macro_rules! optimize_terminator {
+                ($kind:expr) => {
+                    let kind = $kind;
+                    debug!(new_terminator = ?kind);
+                    optimized_terminator =
+                        mir::Terminator { source_info: terminator.source_info, kind };
+                    terminator = &optimized_terminator;
+                };
+            }
+
+            match enum_layout.layout.variants {
+                _ if enum_layout.abi.is_uninhabited() => {
+                    debug!("uninhabited");
+
+                    optimize_terminator!(mir::TerminatorKind::Unreachable);
+                }
+                abi::Variants::Single { index } => {
+                    let discr = discr_for_variant(index);
+                    debug!(?index, ?discr, "single variant");
+
+                    let target = targets
+                        .iter()
+                        .find(|(val, _)| *val == discr)
+                        .map_or(targets.otherwise(), |(_, bb)| bb);
+
+                    optimize_terminator!(mir::TerminatorKind::Goto { target });
+                }
+                abi::Variants::Multiple { ref variants, ref tag_encoding, .. } => {
+                    debug!(variants = variants.len(), "multiple variants");
+
+                    fn filter_targets(
+                        targets: &mir::SwitchTargets,
+                        mut predicate: impl FnMut(u128) -> bool,
+                    ) -> Option<mir::SwitchTargets> {
+                        let first_culled = targets.iter().position(|(val, _)| !predicate(val))?;
+
+                        Some(mir::SwitchTargets::new(
+                            targets.iter().take(first_culled).chain(
+                                targets
+                                    .iter()
+                                    .skip(first_culled + 1)
+                                    .filter(|(val, _)| predicate(*val)),
+                            ),
+                            targets.otherwise(),
+                        ))
+                    }
+
+                    let filtered_targets = match tag_encoding {
+                        abi::TagEncoding::Niche { .. } => {
+                            debug!("niche tag");
+
+                            filter_targets(targets, |index| {
+                                index < variants.len() as u128
+                                    && !variants[abi::VariantIdx::from(index as u32)]
+                                        .abi
+                                        .is_uninhabited()
+                            })
+                        }
+                        abi::TagEncoding::Direct => {
+                            debug!("direct tag");
+
+                            // FIXME(nox): Use `AdtDef::discriminants`.
+                            let inhabited_discriminants = variants
+                                .indices()
+                                .filter_map(|index| {
+                                    if variants[index].abi.is_uninhabited() {
+                                        return None;
+                                    }
+                                    Some(discr_for_variant(index))
+                                })
+                                .collect::<Vec<_>>();
+                            filter_targets(targets, |discr| {
+                                inhabited_discriminants.contains(&discr)
+                            })
+                        }
+                    };
+
+                    if let Some(filtered_targets) = filtered_targets {
+                        optimize_terminator!(mir::TerminatorKind::SwitchInt {
+                            discr: discr.clone(),
+                            switch_ty: switch_ty,
+                            targets: filtered_targets,
+                        });
+                    }
+                }
+            }
+        }
+
         self.codegen_terminator(bx, bb, terminator);
     }
 
+    fn get_discriminant_switch_block_parts<'t>(
+        &mut self,
+        statements: &'tcx [mir::Statement<'tcx>],
+        terminator: &'t mir::Terminator<'tcx>,
+    ) -> Option<(Ty<'tcx>, &'t mir::Operand<'tcx>, Ty<'tcx>, &'t SwitchTargets)> {
+        if !self.cx.tcx().sess.opts.debugging_opts.opt_switch_monomorphizing {
+            return None;
+        }
+
+        let (discr, switch_ty, targets) = terminator.kind.as_switch()?;
+        let discr_place = discr.place()?;
+        let last = statements.last()?;
+        let (lhs, rhs) = last.kind.as_assign()?;
+
+        if *lhs != discr_place {
+            return None;
+        };
+
+        let &mir::Rvalue::Discriminant(enum_place) = rhs else { return None };
+
+        assert_eq!(switch_ty, self.monomorphized_place_ty(discr_place.as_ref()));
+
+        let enum_ty = self.monomorphized_place_ty(enum_place.as_ref());
+
+        Some((enum_ty, discr, switch_ty, targets))
+    }
+
     fn codegen_terminator(
         &mut self,
         mut bx: Bx,
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 29c4f78b42e..a80a272289e 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -761,6 +761,7 @@ fn test_debugging_options_tracking_hash() {
     tracked!(no_profiler_runtime, true);
     tracked!(no_unique_section_names, true);
     tracked!(normalize_docs, true);
+    tracked!(opt_switch_monomorphizing, true);
     tracked!(osx_rpath_install_name, true);
     tracked!(panic_abort_tests, true);
     tracked!(panic_in_drop, PanicStrategy::Abort);
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 6bea1adfaea..d40f7b77b94 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1261,6 +1261,8 @@ options! {
         "prevent automatic injection of the profiler_builtins crate"),
     normalize_docs: bool = (false, parse_bool, [TRACKED],
         "normalize associated items in rustdoc when generating documentation"),
+    opt_switch_monomorphizing: bool = (true, parse_bool, [TRACKED],
+        "optimize switch monomorphizing"),
     osx_rpath_install_name: bool = (false, parse_bool, [TRACKED],
         "pass `-install_name @rpath/...` to the macOS linker (default: no)"),
     panic_abort_tests: bool = (false, parse_bool, [TRACKED],
diff --git a/src/test/codegen/opt-switch-monomorphizing/intrinsic.rs b/src/test/codegen/opt-switch-monomorphizing/intrinsic.rs
new file mode 100644
index 00000000000..5fc37b709d4
--- /dev/null
+++ b/src/test/codegen/opt-switch-monomorphizing/intrinsic.rs
@@ -0,0 +1,194 @@
+// revisions: CHECK-BASE CHECK-OPT
+// compile-flags: -C no-prepopulate-passes -C opt-level=0 -Z mir-opt-level=1
+//[CHECK-BASE] compile-flags: -Z opt-switch-monomorphizing=off
+//[CHECK-OPT] compile-flags: -Z opt-switch-monomorphizing=on
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+#![feature(never_type)]
+
+use std::intrinsics::discriminant_value;
+use std::num::NonZeroUsize;
+
+pub enum BothEmpty {
+    Left(!),
+    Right(!),
+}
+
+// CHECK-LABEL: @match_both_empty
+#[no_mangle]
+pub fn match_both_empty(e: BothEmpty) -> u8 {
+    // CHECK: %e =
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] undef, label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT:    i[[SIZE]] 0
+    // CHECK-BASE-NEXT:    i[[SIZE]] 1
+    // CHECK-BASE-NEXT:    i[[SIZE]] 2
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: unreachable
+    match discriminant_value(&e) {
+        0 => 100,
+        1 => 101,
+        2 => 102,
+        _ => 103,
+    }
+}
+
+pub enum EmptyOrBool {
+    Empty(!),
+    Bool(bool),
+}
+
+// CHECK-LABEL: @match_empty_or_bool
+#[no_mangle]
+pub fn match_empty_or_bool(e: EmptyOrBool) -> u8 {
+    // CHECK: store i8 %[[D:[0-9]+]], i8* %e
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] 1, label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0
+    // CHECK-BASE-NEXT: i[[SIZE]] 1, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: i[[SIZE]] 2
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: br label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 101, i8* %1
+    match discriminant_value(&e) {
+        0 => 100,
+        1 => 101,
+        2 => 102,
+        _ => 103,
+    }
+}
+
+pub enum BoolOrEmpty {
+    Bool(bool),
+    Empty(!),
+}
+
+// CHECK-LABEL: @match_bool_or_empty
+#[no_mangle]
+pub fn match_bool_or_empty(e: BoolOrEmpty) -> u8 {
+    // CHECK: store i8 %[[D:[0-9]+]], i8* %e
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] 0, label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0, label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: i[[SIZE]] 1
+    // CHECK-BASE-NEXT: i[[SIZE]] 2
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: br label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK: [[L]]:
+    // CHECK-NEXT: store i8 100, i8* %1
+    match discriminant_value(&e) {
+        0 => 100,
+        1 => 101,
+        2 => 102,
+        _ => 103,
+    }
+}
+
+pub enum UninhabitedUsizeOrBool {
+    Usize(usize, !),
+    Bool(bool),
+}
+
+// CHECK-LABEL: @match_uninhabited_usize_or_bool
+#[no_mangle]
+pub fn match_uninhabited_usize_or_bool(e: UninhabitedUsizeOrBool) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i8, i8* %{{[0-9]+}}
+    // CHECK-NEXT: %_[[D:[0-9]+]] = zext i8 %[[TAG]] to i[[SIZE:[0-9]+]]
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] %_[[D]], label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0
+    // CHECK-BASE-NEXT: i[[SIZE]] 1, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: i[[SIZE]] 2
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 1
+    // CHECK-OPT-NEXT: br i1 %[[CMP]], label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 101, i8* %1
+    match discriminant_value(&e) {
+        0 => 100,
+        1 => 101,
+        2 => 102,
+        _ => 103,
+    }
+}
+
+pub enum BoolOrUninhabitedUsize {
+    Bool(bool),
+    Usize(usize, !),
+}
+
+// CHECK-LABEL: @match_bool_or_uninhabited_nonzero_usize
+#[no_mangle]
+pub fn match_bool_or_uninhabited_nonzero_usize(e: BoolOrUninhabitedUsize) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i8, i8* %{{[0-9]+}}
+    // CHECK-NEXT: %_[[D:[0-9]+]] = zext i8 %[[TAG]] to i[[SIZE:[0-9]+]]
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] %_[[D]], label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0, label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: i[[SIZE]] 1
+    // CHECK-BASE-NEXT: i[[SIZE]] 2
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 0
+    // CHECK-OPT-NEXT: br i1 %[[CMP]], label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK: [[L]]:
+    // CHECK-NEXT: store i8 100, i8* %1
+    match discriminant_value(&e) {
+        0 => 100,
+        1 => 101,
+        2 => 102,
+        _ => 103,
+    }
+}
+
+pub enum UninhabitedNonZeroUsizeOrUnit {
+    Usize(NonZeroUsize, !),
+    Unit,
+}
+
+// CHECK-LABEL: @match_uninhabited_non_zero_usize_or_unit
+#[no_mangle]
+pub fn match_uninhabited_non_zero_usize_or_unit(e: UninhabitedNonZeroUsizeOrUnit) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i[[SIZE:[0-9]+]], i[[SIZE]]* %{{[0-9]+}}
+    // CHECK-NEXT: %[[TMP:[0-9]+]] = icmp eq i[[SIZE]] %[[TAG]], 0
+    // CHECK-NEXT: %_[[D:[0-9]+]] = select i1 %[[TMP]], i[[SIZE]] 1, i[[SIZE]] 0
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] %_[[D]], label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0
+    // CHECK-BASE-NEXT: i[[SIZE]] 1, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: i[[SIZE]] 2
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 1
+    // CHECK-OPT-NEXT: br i1 %[[CMP]], label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 101, i8* %1
+    match discriminant_value(&e) {
+        0 => 100,
+        1 => 101,
+        2 => 102,
+        _ => 103,
+    }
+}
+
+pub enum UnitOrUninhabitedNonZeroUsize {
+    Unit,
+    Usize(NonZeroUsize, !),
+}
+
+// CHECK-LABEL: @match_unit_or_uninhabited_non_zero_usize
+#[no_mangle]
+pub fn match_unit_or_uninhabited_non_zero_usize(e: UnitOrUninhabitedNonZeroUsize) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i[[SIZE:[0-9]+]], i[[SIZE]]* %{{[0-9]+}}
+    // CHECK-NEXT: %[[TMP:[0-9]+]] = icmp eq i[[SIZE]] %[[TAG]], 0
+    // CHECK-NEXT: %_[[D:[0-9]+]] = select i1 %[[TMP]], i[[SIZE]] 0, i[[SIZE]] 1
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] %_[[D]], label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: i[[SIZE]] 1
+    // CHECK-BASE-NEXT: i[[SIZE]] 2
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 0
+    // CHECK-OPT-NEXT: br i1 %[[CMP]], label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 100, i8* %1
+    match discriminant_value(&e) {
+        0 => 100,
+        1 => 101,
+        2 => 102,
+        _ => 103,
+    }
+}
diff --git a/src/test/codegen/opt-switch-monomorphizing/match.rs b/src/test/codegen/opt-switch-monomorphizing/match.rs
new file mode 100644
index 00000000000..417cad2ecba
--- /dev/null
+++ b/src/test/codegen/opt-switch-monomorphizing/match.rs
@@ -0,0 +1,313 @@
+// revisions: CHECK-BASE CHECK-OPT
+// compile-flags: -C no-prepopulate-passes -Z mir-opt-level=0
+//[CHECK-BASE] compile-flags: -Z opt-switch-monomorphizing=off
+//[CHECK-OPT] compile-flags: -Z opt-switch-monomorphizing=on
+
+#![crate_type = "lib"]
+#![feature(never_type)]
+
+use std::num::NonZeroUsize;
+
+// CHECK-LABEL: @match_never
+#[no_mangle]
+pub fn match_never(e: !) -> u8 {
+    // CHECK-NOT: switch
+    // CHECK: unreachable
+    match e {}
+}
+
+pub enum BothEmpty {
+    Left(!),
+    Right(!),
+}
+
+// CHECK-LABEL: @match_both_empty
+#[no_mangle]
+pub fn match_both_empty(e: BothEmpty) -> u8 {
+    // CHECK: %e =
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] undef, label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT:    i[[SIZE]] 0
+    // CHECK-BASE-NEXT:    i[[SIZE]] 1
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: unreachable
+    match e {
+        BothEmpty::Left(_) => 100,
+        BothEmpty::Right(_) => 101,
+    }
+}
+
+pub enum BothEmptyInts {
+    Bool(bool, !),
+    Usize(usize, !),
+}
+
+// CHECK-LABEL: @match_both_empty_ints
+#[no_mangle]
+pub fn match_both_empty_ints(e: BothEmptyInts) -> u8 {
+    // CHECK: %e = alloca %BothEmptyInts
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] undef, label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT:    i[[SIZE]] 0
+    // CHECK-BASE-NEXT:    i[[SIZE]] 1
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: unreachable
+    match e {
+        BothEmptyInts::Bool(_, _) => 100,
+        BothEmptyInts::Usize(_, _) => 101,
+    }
+}
+
+pub enum BothEmptyWithNiche {
+    Unit(!),
+    Usize(NonZeroUsize, !),
+}
+
+// CHECK-LABEL: @match_both_empty_with_niche
+#[no_mangle]
+pub fn match_both_empty_with_niche(e: BothEmptyWithNiche) -> u8 {
+    // CHECK: %e =
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] undef, label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT:    i[[SIZE]] 0
+    // CHECK-BASE-NEXT:    i[[SIZE]] 1
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: unreachable
+    match e {
+        BothEmptyWithNiche::Unit(_) => 100,
+        BothEmptyWithNiche::Usize(_, _) => 101,
+    }
+}
+
+pub enum EmptyOrBool {
+    Empty(!),
+    Bool(bool),
+}
+
+// CHECK-LABEL: @match_empty_or_bool
+#[no_mangle]
+pub fn match_empty_or_bool(e: EmptyOrBool) -> u8 {
+    // CHECK: store i8 %{{[0-9]+}}, i8* %e
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] 1, label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0
+    // CHECK-BASE-NEXT: i[[SIZE]] 1, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: br label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 101, i8* %1
+    match e {
+        EmptyOrBool::Empty(_) => 100,
+        EmptyOrBool::Bool(_) => 101,
+    }
+}
+
+// CHECK-LABEL: @match_empty_or_bool_otherwise
+#[no_mangle]
+pub fn match_empty_or_bool_otherwise(e: EmptyOrBool) -> u8 {
+    // CHECK: store i8 %{{[0-9]+}}, i8* %e
+    // CHECK-BASE-NEXT: br i1 false, label %{{[a-zA-Z0-9_]+}}, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-OPT-NEXT: br label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 101, i8* %1
+    match e {
+        EmptyOrBool::Empty(_) => 100,
+        _ => 101,
+    }
+}
+
+pub enum BoolOrEmpty {
+    Bool(bool),
+    Empty(!),
+}
+
+// CHECK-LABEL: @match_bool_or_empty
+#[no_mangle]
+pub fn match_bool_or_empty(e: BoolOrEmpty) -> u8 {
+    // CHECK: store i8 %{{[0-9]+}}, i8* %e
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] 0, label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0, label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: i[[SIZE]] 1
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: br label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK: [[L]]:
+    // CHECK-NEXT: store i8 100, i8* %1
+    match e {
+        BoolOrEmpty::Bool(_) => 100,
+        BoolOrEmpty::Empty(_) => 101,
+    }
+}
+
+// CHECK-LABEL: @match_bool_or_empty_otherwise
+#[no_mangle]
+pub fn match_bool_or_empty_otherwise(e: BoolOrEmpty) -> u8 {
+    // CHECK: store i8 %{{[0-9]+}}, i8* %e
+    // CHECK-BASE-NEXT: br i1 false, label %{{[a-zA-Z0-9_]+}}, label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK-OPT-NEXT: br label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK: [[L]]:
+    // CHECK-NEXT: store i8 100, i8* %1
+    match e {
+        BoolOrEmpty::Empty(_) => 101,
+        _ => 100,
+    }
+}
+
+pub enum UninhabitedUsizeOrBool {
+    Usize(usize, !),
+    Bool(bool),
+}
+
+// CHECK-LABEL: @match_uninhabited_usize_or_bool
+#[no_mangle]
+pub fn match_uninhabited_usize_or_bool(e: UninhabitedUsizeOrBool) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i8, i8* %{{[0-9]+}}
+    // CHECK-NEXT: %_[[D:[0-9]+]] = zext i8 %[[TAG]] to i[[SIZE:[0-9]+]]
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] %_[[D]], label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0
+    // CHECK-BASE-NEXT: i[[SIZE]] 1, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 1
+    // CHECK-OPT-NEXT: br i1 %[[CMP]], label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 101, i8* %1, align 1
+    match e {
+        UninhabitedUsizeOrBool::Usize(_, _) => 100,
+        UninhabitedUsizeOrBool::Bool(_) => 101,
+    }
+}
+
+// CHECK-LABEL: @match_uninhabited_usize_or_bool_otherwise
+#[no_mangle]
+pub fn match_uninhabited_usize_or_bool_otherwise(e: UninhabitedUsizeOrBool) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i8, i8* %{{[0-9]+}}
+    // CHECK-NEXT: %_[[D:[0-9]+]] = zext i8 %[[TAG]] to i[[SIZE:[0-9]+]]
+    // CHECK-BASE-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 0
+    // CHECK-BASE-NEXT: br i1 %[[CMP]], label %{{[a-zA-Z0-9_]+}}, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-OPT-NEXT: br label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 101, i8* %1, align 1
+    match e {
+        UninhabitedUsizeOrBool::Usize(_, _) => 100,
+        _ => 101,
+    }
+}
+
+pub enum BoolOrUninhabitedUsize {
+    Bool(bool),
+    Usize(usize, !),
+}
+
+// CHECK-LABEL: @match_bool_or_uninhabited_nonzero_usize
+#[no_mangle]
+pub fn match_bool_or_uninhabited_nonzero_usize(e: BoolOrUninhabitedUsize) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i8, i8* %{{[0-9]+}}
+    // CHECK-NEXT: %_[[D:[0-9]+]] = zext i8 %[[TAG]] to i[[SIZE:[0-9]+]]
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] %_[[D]], label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0, label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: i[[SIZE]] 1
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 0
+    // CHECK-OPT-NEXT: br i1 %[[CMP]], label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK: [[L]]:
+    // CHECK-NEXT: store i8 100, i8* %1, align 1
+    match e {
+        BoolOrUninhabitedUsize::Bool(_) => 100,
+        BoolOrUninhabitedUsize::Usize(_, _) => 101,
+    }
+}
+
+// CHECK-LABEL: @match_bool_or_uninhabited_nonzero_usize_otherwise
+#[no_mangle]
+pub fn match_bool_or_uninhabited_nonzero_usize_otherwise(e: BoolOrUninhabitedUsize) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i8, i8* %{{[0-9]+}}
+    // CHECK-NEXT: %_[[D:[0-9]+]] = zext i8 %[[TAG]] to i[[SIZE:[0-9]+]]
+    // CHECK-BASE-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 1
+    // CHECK-BASE-NEXT: br i1 %[[CMP]], label %{{[a-zA-Z0-9_]+}}, label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK-OPT-NEXT: br label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK: [[L]]:
+    // CHECK-NEXT: store i8 100, i8* %1, align 1
+    match e {
+        BoolOrUninhabitedUsize::Usize(_, _) => 101,
+        _ => 100,
+    }
+}
+
+pub enum UninhabitedNonZeroUsizeOrUnit {
+    Usize(NonZeroUsize, !),
+    Unit,
+}
+
+// CHECK-LABEL: @match_uninhabited_non_zero_usize_or_unit
+#[no_mangle]
+pub fn match_uninhabited_non_zero_usize_or_unit(e: UninhabitedNonZeroUsizeOrUnit) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i[[SIZE:[0-9]+]], i[[SIZE]]* %{{[0-9]+}}
+    // CHECK-NEXT: %[[TMP:[0-9]+]] = icmp eq i[[SIZE]] %[[TAG]], 0
+    // CHECK-NEXT: %_[[D:[0-9]+]] = select i1 %[[TMP]], i[[SIZE]] 1, i[[SIZE]] 0
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] %_[[D]], label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0
+    // CHECK-BASE-NEXT: i[[SIZE]] 1, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 1
+    // CHECK-OPT-NEXT: br i1 %[[CMP]], label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 101, i8* %1, align 1
+    match e {
+        UninhabitedNonZeroUsizeOrUnit::Usize(_, _) => 100,
+        UninhabitedNonZeroUsizeOrUnit::Unit => 101,
+    }
+}
+
+// CHECK-LABEL: @match_uninhabited_non_zero_usize_or_unit_otherwise
+#[no_mangle]
+pub fn match_uninhabited_non_zero_usize_or_unit_otherwise(e: UninhabitedNonZeroUsizeOrUnit) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i[[SIZE:[0-9]+]], i[[SIZE]]* %{{[0-9]+}}
+    // CHECK-NEXT: %[[TMP:[0-9]+]] = icmp eq i[[SIZE]] %[[TAG]], 0
+    // CHECK-NEXT: %_[[D:[0-9]+]] = select i1 %[[TMP]], i[[SIZE]] 1, i[[SIZE]] 0
+    // CHECK-BASE-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 0
+    // CHECK-BASE-NEXT: br i1 %[[CMP]], label %{{[a-zA-Z0-9_]+}}, label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK-OPT-NEXT: br label %[[R:[a-zA-Z0-9_]+]]
+    // CHECK: [[R]]:
+    // CHECK-NEXT: store i8 101, i8* %1, align 1
+    match e {
+        UninhabitedNonZeroUsizeOrUnit::Usize(_, _) => 100,
+        _ => 101,
+    }
+}
+
+pub enum UnitOrUninhabitedNonZeroUsize {
+    Unit,
+    Usize(NonZeroUsize, !),
+}
+
+// CHECK-LABEL: @match_unit_or_uninhabited_non_zero_usize
+#[no_mangle]
+pub fn match_unit_or_uninhabited_non_zero_usize(e: UnitOrUninhabitedNonZeroUsize) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i[[SIZE:[0-9]+]], i[[SIZE]]* %{{[0-9]+}}
+    // CHECK-NEXT: %[[TMP:[0-9]+]] = icmp eq i[[SIZE]] %[[TAG]], 0
+    // CHECK-NEXT: %_[[D:[0-9]+]] = select i1 %[[TMP]], i[[SIZE]] 0, i[[SIZE]] 1
+    // CHECK-BASE-NEXT: switch i[[SIZE:[0-9]+]] %_[[D]], label %{{[a-zA-Z0-9_]+}} [
+    // CHECK-BASE-NEXT: i[[SIZE]] 0, label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK-BASE-NEXT: i[[SIZE]] 1
+    // CHECK-BASE-NEXT: ]
+    // CHECK-OPT-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 0
+    // CHECK-OPT-NEXT: br i1 %[[CMP]], label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK: [[L]]:
+    // CHECK-NEXT: store i8 100, i8* %1, align 1
+    match e {
+        UnitOrUninhabitedNonZeroUsize::Unit => 100,
+        UnitOrUninhabitedNonZeroUsize::Usize(_, _) => 101,
+    }
+}
+
+// CHECK-LABEL: @match_unit_or_uninhabited_non_zero_usize_otherwise
+#[no_mangle]
+pub fn match_unit_or_uninhabited_non_zero_usize_otherwise(e: UnitOrUninhabitedNonZeroUsize) -> u8 {
+    // CHECK: %[[TAG:[0-9]+]] = load i[[SIZE:[0-9]+]], i[[SIZE]]* %{{[0-9]+}}
+    // CHECK-NEXT: %[[TMP:[0-9]+]] = icmp eq i[[SIZE]] %[[TAG]], 0
+    // CHECK-NEXT: %_[[D:[0-9]+]] = select i1 %[[TMP]], i[[SIZE]] 0, i[[SIZE]] 1
+    // CHECK-BASE-NEXT: %[[CMP:[0-9]+]] = icmp eq i[[SIZE]] %_[[D]], 1
+    // CHECK-BASE-NEXT: br i1 %[[CMP]], label %{{[a-zA-Z0-9_]+}}, label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK-OPT-NEXT: br label %[[L:[a-zA-Z0-9_]+]]
+    // CHECK: [[L]]:
+    // CHECK-NEXT: store i8 100, i8* %1, align 1
+    match e {
+        UnitOrUninhabitedNonZeroUsize::Usize(_, _) => 101,
+        _ => 100,
+    }
+}