[Bug c/85362] New: unnecessary checks with -fsanitize=object-size and non-int indices

["breiten at lexmark dot com" <gcc-bugzilla at gcc dot gnu dot org>]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85362

            Bug ID: 85362
           Summary: unnecessary checks with -fsanitize=object-size and
                    non-int indices
           Product: gcc
           Version: 7.2.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: c
          Assignee: unassigned at gcc dot gnu.org
          Reporter: breiten at lexmark dot com
  Target Milestone: ---

Created attachment 43915
  --> https://gcc.gnu.org/bugzilla/attachment.cgi?id=43915&action=edit
example .c file - check for invalid index optimized out of foo() only

The following code, when compiled with gcc-6.4 or gcc-7.2 and
-fsanitize=object-size and -O2 or higher creates unnecessary code to ensure a
valid index.  I'm configured for aarch64, but also see this with armv7.

When indexing a 256-element array with either a uint8_t variable or an integer
variable shifted right by 24 or more, gcc inserts code to ensure a valid index.
 That code DOES get optimized out when using an integer variable with a smaller
or variable shift count and an explicit mask with 0xff.

With the index clearly being an unsigned 8-bit value, the check for > 256
should not be needed.  Even explicitly and'ing with 0xff does not cause the
check to get optimized out when using an 8-bit datatype.

lut[(*pi >> 23) & 0xff]);  // good
lut[(*pi >> 24) & 0xff]);  // extra check
lut[(*pi >> 25) & 0xff]);  // extra check
lut[(*pi >> 24) & 0x1]);   // extra check
lut[*pb]                   // extra check
lut[*pb & 0xff];           // extra check
lut[*pb >> 7];             // extra check
lut[*pb >> 8];             // good - loads first element

/bonus/scratch/gcc64/poky/proj/tmp/work/aarch64-poky-linux/graph/*/recipe-sysroot-native/usr/bin/aarch64-poky-linux/aarch64-poky-linux-gcc
 -mcpu=cortex-a53  
--sysroot=/bonus/scratch/gcc64/poky/proj/tmp/work/aarch64-poky-linux/graph/*/recipe-sysroot
-S -O4  -fsanitize=object-size  -fno-sanitize-recover
-fsanitize-undefined-trap-on-error -Wall -Wextra   ~/sanitize.c

gcc-7.2 assembly output follows:
        .arch armv8-a+crc
        .file   "sanitize.c"
        .text
        .align  2
        .p2align 4,,15
        .global foo
        .type   foo, %function
foo:
        ldrb    w1, [x0, 1]
        adrp    x0, lut
        add     x0, x0, :lo12:lut
        ldr     w0, [x0, x1, lsl 2]
        ret
        .size   foo, .-foo
        .align  2
        .p2align 4,,15
        .global bar
        .type   bar, %function
bar:
        ldrb    w2, [x0, 3]
        adrp    x1, lut
        add     x0, x1, :lo12:lut
        add     x3, x0, x2, lsl 2
        sub     x0, x3, x0
        add     x0, x0, 4
        cmp     x0, 1024
        bhi     .L6
.L4:
        add     x1, x1, :lo12:lut
        ldr     w0, [x1, x2, lsl 2]
        ret
.L6:
        add     x0, x3, x0
        cmp     x3, x0
        bhi     .L4
        brk #1000
        .size   bar, .-bar
        .align  2
        .p2align 4,,15
        .global baz
        .type   baz, %function
baz:
        ldrb    w2, [x0]
        adrp    x1, lut
        add     x0, x1, :lo12:lut
        add     x3, x0, x2, uxtb 2
        sub     x0, x3, x0
        add     x0, x0, 4
        cmp     x0, 1024
        bhi     .L10
.L8:
        add     x1, x1, :lo12:lut
        ldr     w0, [x1, x2, lsl 2]
        ret
.L10:
        add     x0, x3, x0
        cmp     x3, x0
        bhi     .L8
        brk #1000
        .size   baz, .-baz
        .ident  "GCC: (GNU) 7.2.0"
        .section        .note.GNU-stack,"",@progbits