/*- * Copyright (c) 2003 Peter Wemm. * Copyright (c) 1992 Terrence R. Lambert. * Copyright (c) 1982, 1987, 1990 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91 */ /*- * Copyright (c) 2011, Sandvine Incorporated ULC. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_msgbuf.h" #include "opt_rasum.h" #include #include #include #include #include #include #include #include #include #include #include #include #ifdef RASUM_MEMTEST_KERNEL #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SMP #include #endif #include #include vm_paddr_t phys_avail[PHYSMAP_SIZE + 2]; vm_paddr_t dump_avail[PHYSMAP_SIZE + 2]; /* must be 2 less so 0 0 can signal end of chunks */ #define PHYS_AVAIL_ARRAY_END ((sizeof(phys_avail) / sizeof(phys_avail[0])) - 2) #define DUMP_AVAIL_ARRAY_END ((sizeof(dump_avail) / sizeof(dump_avail[0])) - 2) /* List of found faulty address ranges. */ static vm_paddr_t phys_faulty[PHYSMAP_SIZE + 2]; #define PHYS_ERR_ARRAY_END ((sizeof(phys_faulty) / sizeof(phys_faulty[0])) - 2) static u_long memtest_loops = 1; #define MEMTEST_COMPLETION_BOOT 1 #define MEMTEST_COMPLETION_REBOOT 2 #define MEMTEST_COMPLETION_WAIT 3 #define MEMTEST_COMPLETION_USER_ABORT (1ul << 30) #define MEMTEST_COMPLETION_TIMEOUT (1ul << 31) #define MEMTEST_COMPLETION_MODE_MASK (MEMTEST_COMPLETION_BOOT | \ MEMTEST_COMPLETION_REBOOT | \ MEMTEST_COMPLETION_WAIT ) #define MEMTEST_COMPLETION_END_MASK (MEMTEST_COMPLETION_USER_ABORT | \ MEMTEST_COMPLETION_TIMEOUT) /* Time difference in seconds when we should report on a long-running test. */ #define MEMTEST_INTERIM_REPORT_DT 600 /* Flags for mapping pages. */ #define MEMTEST_PG_NOCACHE 1 #define MEMTEST_PG_PAGEFILL 2 #define MEMTEST_PG_SMBIOS 3 static __inline uint64_t memtest_get_pg(uint64_t idx) { switch (idx) { case MEMTEST_PG_SMBIOS: case MEMTEST_PG_PAGEFILL: return (PG_V | PG_RW | pmap_cache_bits(PAT_WRITE_COMBINING, 0)); #ifdef __notyet__ /* * Consider benchmarking a clflush() version (on HEAD) with all * infrastructure in place. */ case MEMTEST_PG_CACHE_CLFLUSH: return (PG_V | PG_RW); #endif default: case MEMTEST_PG_NOCACHE: return (PG_V | PG_RW | PG_N); } } /* The order the tests appear here mandates the order in which they are run. */ #define MEMTEST_TEST_WALK0 0 #define MEMTEST_TEST_WALK1 1 #define MEMTEST_TEST_ADDRWALK 2 #define MEMTEST_TEST_ALLONES 3 #define MEMTEST_TEST_ALLZEROS 4 #define MEMTEST_TEST_USERPATTERN 5 #define MEMTEST_TEST_INVERSION 6 #define MEMTEST_TEST_FREEBSD_CLASSIC 20 #define MEMTEST_TEST_MAX_SHIFT 20 #ifdef RASUM_MEMTEST_KERNEL #define MEMTEST_TEST_NO_RASUM 29 /* Disable RASUM support. */ #else /* Redefine for MEMTEST_TEST_ALL_MASK. */ #define MEMTEST_TEST_NO_RASUM MEMTEST_TEST_WALK0 #endif #define MEMTEST_TEST_NO_SAVE 30 /* Prevent data save/restore. */ #define MEMTEST_TEST_NO_SMBIOS 31 /* Disable SMBIOS support. */ /* We ignore the classic FreeBSD test for our valid list of all tests. */ #define MEMTEST_TEST_ALL_MASK ( \ ((u_long)1 << MEMTEST_TEST_WALK0) | \ ((u_long)1 << MEMTEST_TEST_WALK1) | \ ((u_long)1 << MEMTEST_TEST_ADDRWALK) | \ ((u_long)1 << MEMTEST_TEST_ALLONES) | \ ((u_long)1 << MEMTEST_TEST_ALLZEROS) | \ ((u_long)1 << MEMTEST_TEST_USERPATTERN) | \ ((u_long)1 << MEMTEST_TEST_INVERSION) | \ ((u_long)1 << MEMTEST_TEST_NO_RASUM) | \ ((u_long)1 << MEMTEST_TEST_NO_SAVE) | \ ((u_long)1 << MEMTEST_TEST_NO_SMBIOS) ) struct memtest_test { const char * name; /* Test name. */ const char * descr; /* Test description. */ u_long iter; /* # iteration this test ran. */ uint64_t err_iter; /* Errors in this iteration. */ uint64_t err_total; /* Errors in total. */ uint64_t err_count; /* # iterations with errors. */ uint64_t bw_iter_ticks; /* Bandwidth this iteration, */ uint64_t bw_iter_bytes; /* ticks and bytes. */ uint64_t bw_total_ticks; /* Bandwidth total, */ uint64_t bw_total_bytes; /* ticks and bytes. */ uint64_t cache_bits; /* PG/PAT bits. */ uint64_t (*tfunc)(uint64_t, struct memtest_test *); uint64_t (*lfunc)(struct memtest_test *, vm_paddr_t, void *, uintptr_t); int (*paselect)(vm_paddr_t, vm_paddr_t, vm_paddr_t); }; struct memtest_meta { u_long tests; u_long completion_mode; u_long flags; u_long iter; u_long interim_dt; /* MEMTEST_INTERIM_REPORT_DT */ u_long time_limit; uint64_t upattern; uint64_t start; uint64_t end; uint64_t rdtsc_div; uint32_t smbios_addr; uint16_t smbios_len; uint16_t smbios_count; int physfaulty_idx; int physmap_idx; vm_paddr_t * physmap; u_int64_t first; quad_t dcons_addr; quad_t dcons_size; vm_paddr_t addrwalk[sizeof(vm_paddr_t)*8]; int addrwalk_idx; }; static struct memtest_meta memtest_meta = { .rdtsc_div = 1, .addrwalk_idx = -1, }; static void memtest_phys_faulty_add(vm_paddr_t addr) { int i, j; /* Mask off any bits we might have set for page 0. */ addr = trunc_page(addr); for (i = 0; i < memtest_meta.physfaulty_idx; i += 2) { if (addr < phys_faulty[i] && addr > phys_faulty[i+1]) continue; /* We know that addr is faulty or it is a scucessive page. */ if (addr != phys_faulty[i+1]) return; phys_faulty[i+1] += PAGE_SIZE; /* Check if we can merge two (now overlapping) areas. */ if ((i+3) < PHYS_ERR_ARRAY_END && phys_faulty[i+1] == phys_faulty[i+2]) { phys_faulty[i+1] = phys_faulty[i+3]; for (i += 2; i < memtest_meta.physfaulty_idx; i += 2) { phys_faulty[i] = phys_faulty[i-2]; phys_faulty[i+1] = phys_faulty[i-1]; } memtest_meta.physfaulty_idx -= 2; } return; } /* Insert new region sorted if possible. */ if (memtest_meta.physfaulty_idx < PHYS_ERR_ARRAY_END) { for (i = 0; i < memtest_meta.physfaulty_idx; i += 2) { if (addr > phys_faulty[i]) continue; for (j = i; j < memtest_meta.physfaulty_idx; j += 2) { phys_faulty[j] = phys_faulty[j+2]; phys_faulty[j+1] = phys_faulty[j+3]; } memtest_meta.physfaulty_idx += 2; phys_faulty[i] = addr; phys_faulty[i+1] = addr + PAGE_SIZE; return; } phys_faulty[memtest_meta.physfaulty_idx++] = addr; phys_faulty[memtest_meta.physfaulty_idx++] = addr + PAGE_SIZE; } /* * Else we will ignore any memory beyond that not having space * left and with that will ignore all memory after the last end * address as well. */ } #define memtest_log_event(now, _fmt, ...) \ printf("[%6ju] " _fmt "\n", \ (((now)-memtest_meta.start)/memtest_meta.rdtsc_div), __VA_ARGS__); /* * XXX-BZ when forward porting factor these out to a shared include file along * with sharing some functionality. This is based on * sys/boot/i386/libi386/smbios.c and sys/i386/bios/smbios.c. */ #ifndef SMBIOS_START #define SMBIOS_START 0xf0000 #define SMBIOS_STEP 0x10 #define SMBIOS_OFF 0 #define SMBIOS_LEN 4 #define SMBIOS_SIG "_SM_" static uint8_t smbios_checksum(const uint8_t *cp, uint8_t len) { uint8_t sum; sum = 0x00; for (; len > 0; len--) sum += cp[len-1]; return(sum); } #endif static uint8_t * memtest_smbios_get_addr(uint32_t paddr, size_t len) { static pt_entry_t *pte = NULL; uint8_t *p; if (pte == NULL || paddr < trunc_page(*pte) || paddr > (trunc_page(*pte) + PAGE_MASK)) { /* Need to re-map. */ pte = CMAP2; *pte = trunc_page(paddr) | memtest_get_pg(MEMTEST_PG_SMBIOS); invltlb(); } p = (uint8_t *)CADDR2; p += (paddr - trunc_page(paddr)); /* Make sure we can fulfil size requirements. */ if ((uintptr_t)(p + len - (uintptr_t)CADDR2) > (uintptr_t)(PAGE_MASK)) return (NULL); return (p); } #define SMBIOS_STRUCT_GET_FIELD_VOID(paddr, offset, type, var) \ do { \ uint8_t *_cp; \ \ _cp = memtest_smbios_get_addr((paddr)+(offset), \ sizeof(type)); \ if (_cp == NULL) \ return; \ (var) = *(type *)(_cp); \ } while (0) #define SMBIOS_STRUCT_GET_FIELD(paddr, offset, type, var) \ do { \ uint8_t *_cp; \ \ _cp = memtest_smbios_get_addr((paddr)+(offset), \ sizeof(type)); \ if (_cp == NULL) \ return (-1); \ (var) = *(type *)(_cp); \ } while (0) #define SMBIOS_STRUCT_GET_FIELD_NULL(paddr, offset, type, var) \ do { \ uint8_t *_cp; \ \ _cp = memtest_smbios_get_addr((paddr)+(offset), \ sizeof(type)); \ if (_cp == NULL) \ return (_cp); \ (var) = *(type *)(_cp); \ } while (0) static char * smbios_structure_get_string(uint32_t paddr, uint8_t idx, char *buf, char *sp, size_t len) { uint32_t i, off; char c; /* Skip if string is not available. */ if (idx == 0) return (NULL); /* Skip previous strings. */ off = 0; for (i = 1; i < idx; i++) { do { SMBIOS_STRUCT_GET_FIELD_NULL(paddr, off, char, c); off++; } while (c != '\0'); } /* Add separator between options. */ if (sp != buf && (sp - buf + 2) < len) *sp++ = ',', *sp++ = ' ', *sp = '\0'; /* Found out string. */ while ((sp - buf + 1) < len) { SMBIOS_STRUCT_GET_FIELD_NULL(paddr, off, char, c); *sp = c; if (c == '\0') return (sp); off++; sp++; /* Safety belt. */ *sp = '\0'; } return (NULL); } static void smbios_get_mdh(uint16_t h, char *buf, size_t len) { uint8_t *cp, *s; uint32_t paddr, sta; uint16_t scount, stl; uint16_t i, handle; uint8_t type, hlen, p, pp; char *sp, *v; static uint8_t smbios_mdh_string_numbers[] = { /* Order in which we will fill the string buffer. */ 0x10, /* Device Locator, i.e. P1-DIMM1. */ 0x11, /* Bank Locator, i.e. BANK0. */ 0x18, /* Serial#. */ 0x17, /* Manufacturer. */ 0x1a, /* Part#. */ 0x00 /* End of list. */ }; /* Check if we found useful SMBIOS information. */ if (memtest_meta.smbios_len == 0) return; stl = memtest_meta.smbios_len; sta = paddr = memtest_meta.smbios_addr; scount = memtest_meta.smbios_count; cp = memtest_smbios_get_addr(paddr, 1); for (i = 0; i < scount && cp != NULL && paddr < (sta + stl); i++) { SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 0, uint8_t, type); SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 1, uint8_t, hlen); SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 2, uint16_t, handle); if (handle != h) goto skip; /* Match. */ sp = buf; s = smbios_mdh_string_numbers; while (s && *s != 0x00) { /* * Skip options not available in older versions * of SMBIOS spec. */ if (*s > hlen) { s++; continue; } /* Check for at least 3 characters left. */ if ((sp - buf + 3) > len) return; SMBIOS_STRUCT_GET_FIELD_VOID(paddr, *s, uint8_t, *cp); v = smbios_structure_get_string(paddr + hlen, *cp, buf, sp, len); if (v == NULL) { s++; continue; } sp = v; /* Remove any trailing blanks from string. */ while ((sp - buf) > 1 && *(sp - 1) == ' ') { sp--; *sp = '\0'; } s++; } return; skip: paddr += hlen; /* Skip any strings, etc. to end of structure. */ SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 0, uint8_t, p); SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 1, uint8_t, pp); while (p != 0x00 || pp != 0x00) { paddr++; SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 0, uint8_t, p); SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 1, uint8_t, pp); } paddr += 2; } } static uint32_t smbios_find_memory_device(uint32_t paddr, uint32_t end, vm_paddr_t addr, uint64_t now) { uint16_t handle; uint8_t type, hlen, p, pp; SMBIOS_STRUCT_GET_FIELD(paddr, 0, uint8_t, type); SMBIOS_STRUCT_GET_FIELD(paddr, 1, uint8_t, hlen); SMBIOS_STRUCT_GET_FIELD(paddr, 2, uint16_t, handle); if (type == 20) { /* Memory Device Mapped Address (optional) */ uint64_t s, e; uint16_t mdh, memah; char buf[256]; SMBIOS_STRUCT_GET_FIELD(paddr, 4, uint32_t, s); SMBIOS_STRUCT_GET_FIELD(paddr, 8, uint32_t, e); if (s == 0xffffffff && e == 0xffffffff && hlen > 0x12) { SMBIOS_STRUCT_GET_FIELD(paddr, 0x13, uint64_t, s); SMBIOS_STRUCT_GET_FIELD(paddr, 0x1b, uint64_t, e); } else if (s != 0xffffffff && e != 0xffffffff) { /* Values are in kB. */ s <<= 10; e <<= 10; /* Make sure end address is not masked 1k. */ e |= ((1 << 10) - 1); } else { /* Invalid structure. Ignore. */ goto skip; } /* Memory Device (17) */ SMBIOS_STRUCT_GET_FIELD(paddr, 0x0c, uint16_t, mdh); /* Memory Array Mapped Address (19) */ SMBIOS_STRUCT_GET_FIELD(paddr, 0x0e, uint16_t, memah); if (addr < s || addr > e) goto skip; /* Match. */ buf[0] = '\0'; smbios_get_mdh(mdh, buf, sizeof(buf)); memtest_log_event(now, " 0x%jx-0x%jx is %s", (uintmax_t)s, (uintmax_t)e, buf); /* End lookup. */ return (-1); } skip: paddr += hlen; /* Skip any strings, etc. to end of structure. */ SMBIOS_STRUCT_GET_FIELD(paddr, 0, uint8_t, p); SMBIOS_STRUCT_GET_FIELD(paddr, 1, uint8_t, pp); while (p != 0x00 || pp != 0x00) { paddr++; SMBIOS_STRUCT_GET_FIELD(paddr, 0, uint8_t, p); SMBIOS_STRUCT_GET_FIELD(paddr, 1, uint8_t, pp); } return (paddr + 2); } static void memtest_log_smbios_memory_device(struct memtest_test *t __unused, vm_paddr_t addr, uint64_t now) { uint8_t *cp; uint16_t i, scount, stl; uint32_t paddr, sta; /* Check if we found useful SMBIOS information. */ if (memtest_meta.smbios_len == 0) return; stl = memtest_meta.smbios_len; sta = paddr = memtest_meta.smbios_addr; scount = memtest_meta.smbios_count; cp = memtest_smbios_get_addr(paddr, 1); for (i = 0; i < scount && cp != NULL && paddr < (sta + stl); i++) paddr = smbios_find_memory_device(paddr, (sta + stl), addr, now); } /* * XXX-BZ eventually we could check the BCD revision and with that major/minor * for special features of modern versions of the spec. For now get basic * information needed. */ static void memtest_find_smbios(void) { char *sysenv; uint8_t *cp; uint32_t smbiosp, sta, paddr; uint16_t i, stl, scount; uint8_t len, vmaj, vmin; memtest_meta.smbios_len = 0; memtest_meta.smbios_addr = 0; memtest_meta.smbios_count = 0; /* * See if loader has set one of the mandatory SMBIOS fields. If not * do not even try to find the SMBIOS location and decode ourself. */ sysenv = getenv("smbios.bios.reldate"); if (sysenv == NULL) return; freeenv(sysenv); smbiosp = SMBIOS_START; while (1) { smbiosp = bios_sigsearch(smbiosp, SMBIOS_SIG, SMBIOS_LEN, SMBIOS_STEP, SMBIOS_OFF); if (smbiosp == 0) return; cp = (uint8_t *)(uintptr_t)BIOS_PADDRTOVADDR(smbiosp); len = *(cp + 0x05); vmaj = *(cp + 0x06); vmin = *(cp + 0x07); /* Work around possible specification error. */ if (vmaj == 2 && vmin == 1 && len == 0x1e) len = 0x1f; if (len != 0x1f) { smbiosp += SMBIOS_STEP; continue; } if ((len = smbios_checksum(cp, len)) != 0) { smbiosp += SMBIOS_STEP; continue; } /* Success we think. */ break; } stl = *(uint16_t *)(cp + 0x16); /* Structure Table Length */ sta = *(uint32_t *)(cp + 0x18); /* Structure Table Address */ scount = *(uint16_t *)(cp + 0x1c); /* # of SMBIOS Structures */ /* * Try to find optional type 20 information. Without that smbios is * no help for us. We will need type 17 in addition but that is * mandatory. */ paddr = sta; cp = memtest_smbios_get_addr(paddr, 1); for (i = 0; i < scount && cp != NULL && paddr < (sta + stl); i++) { uint16_t handle; uint8_t type, hlen, p, pp; SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 0, uint8_t, type); SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 1, uint8_t, hlen); SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 2, uint16_t, handle); if (type == 20) { #ifdef __notyet__ /* XXX-BZ with verbose? */ uint64_t now; now = rdtsc(); memtest_log_start(now, "Found SMBIOS version %u.%u at " "%p, %u structures (IEPS checksum ok)", vmaj, vmin, cq, scount); #endif memtest_meta.smbios_len = stl; memtest_meta.smbios_addr = sta; memtest_meta.smbios_count = scount; break; } paddr += hlen; /* Skip any strings, etc. to end of structure. */ do { SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 0, uint8_t, p); SMBIOS_STRUCT_GET_FIELD_VOID(paddr, 1, uint8_t, pp); } while ((p != 0x00 || pp != 0x00) && ++paddr); paddr += 2; } } static void memtest_log_bad_addr(struct memtest_test *t, vm_paddr_t addr, size_t pwidth, uintptr_t pat, uintptr_t val) { uint64_t now; now = rdtsc(); memtest_log_event(now, "Memtest '%s' FAILURE at 0x%0*jx: " "expected 0x%0*jx got 0x%0*jx", t->name, (int)sizeof(vm_paddr_t)*2, (uintmax_t)addr, (int)pwidth*2, (uintmax_t)pat, (int)pwidth*2, (uintmax_t)val); if ((memtest_meta.flags & ((u_long)1 << MEMTEST_TEST_NO_SMBIOS)) == 0) memtest_log_smbios_memory_device(t, addr, now); #ifdef notyet /* XXX at this point we could lookup ECC, temperature, ... */ #endif t->err_iter++; memtest_phys_faulty_add(addr); } static void memtest_log_bad_data(struct memtest_test *t, vm_paddr_t addr, size_t pwidth, uintptr_t pat, uintptr_t val) { uint64_t now; now = rdtsc(); memtest_log_event(now, "Memtest '%s' data line FAILURE at 0x%0*jx? " "Expected 0x%0*jx got 0x%0*jx", t->name, (int)sizeof(vm_paddr_t)*2, (uintmax_t)addr, (int)pwidth*2, (uintmax_t)pat, (int)pwidth*2, (uintmax_t)val); t->err_iter++; } static int memtest_timelimit_reached(uint64_t now) { u_long dt; /* Check if we recorded it before due to cascading breaks. */ if (memtest_meta.completion_mode & MEMTEST_COMPLETION_TIMEOUT) return (1); if (memtest_meta.time_limit == 0) return (0); dt = (now - memtest_meta.start) / memtest_meta.rdtsc_div; if (dt < memtest_meta.time_limit) return (0); memtest_meta.completion_mode |= MEMTEST_COMPLETION_TIMEOUT; memtest_log_event(now, "Warning: hw.memtest.time_limit=%lu " "reached: %lu", memtest_meta.time_limit, dt); return (1); } static int memtest_user_abort(uint64_t now) { int c; /* Check if we recorded it before due to cascading breaks. */ if (memtest_meta.completion_mode & MEMTEST_COMPLETION_USER_ABORT) return (1); /* Check for user abort. */ c = cncheckc(); if (c == 0x03) { memtest_meta.completion_mode |= MEMTEST_COMPLETION_USER_ABORT; /* Upon user abort, we always want to continue booting. */ memtest_meta.completion_mode &= ~MEMTEST_COMPLETION_MODE_MASK; memtest_meta.completion_mode |= MEMTEST_COMPLETION_BOOT; memtest_log_event(now, "Warning: %s", "memory tests aborted by user. Will boot"); return (1); } #ifdef notyet if (c != -1) /* XXX-BZ maybe with verbose enabled? */ memtest_log_event(now, "Info: %s". "Press CTRL-C to abort"); #endif return (0); } static char memtest_fullpage_save[PAGE_SIZE]; static void memtest_data_save(void *addr, size_t len) { KASSERT(len <= PAGE_SIZE, ("%s: len %zu > PAGE_SIZE", __func__, len)); /* * If we will not continue to boot, we will not save or restore * original page data. Otherwise we add the significant costs to * keep the old msgbuf around possibly etc. */ if (len == 0 || memtest_meta.flags & ((u_long)1 << MEMTEST_TEST_NO_SAVE) || memtest_meta.completion_mode != MEMTEST_COMPLETION_BOOT) return; #ifdef __amd64__ if (len == PAGE_SIZE) pagecopy(addr, memtest_fullpage_save); else #endif bcopy(addr, memtest_fullpage_save, len); } static void memtest_data_restore(void *addr, size_t len) { KASSERT(len <= PAGE_SIZE, ("%s: len %zu > PAGE_SIZE", __func__, len)); /* * If we will not continue to boot, we will not save or restore * original page data. Otherwise we add the significant costs to * keep the old msgbuf around possibly etc. */ if (len == 0 || memtest_meta.flags & ((u_long)1 << MEMTEST_TEST_NO_SAVE) || memtest_meta.completion_mode != MEMTEST_COMPLETION_BOOT) return; return; #ifdef __amd64__ if (len == PAGE_SIZE) pagecopy(memtest_fullpage_save, addr); else #endif bcopy(memtest_fullpage_save, addr, len); } static uint64_t memtest_dataline_walk(struct memtest_test *t, vm_paddr_t pa, void *addr, uintptr_t pat) { uintptr_t sav, tmp, val, x; /* Save original value. */ sav = *(uintptr_t *)addr; #define WRITE_AND_CHECK \ *(volatile uintptr_t *)addr = val; \ tmp = *(volatile uintptr_t *)addr; \ if (val != tmp) \ memtest_log_bad_data(t, pa, sizeof(val), val, tmp) x = ((uintptr_t)1 << (sizeof(val)*8-1)); switch (pat) { case 0: /* Initialize everything to 1 and check. */ val = -1; WRITE_AND_CHECK; /* Move to the left. */ val &= ~(1); while (val != -1) { WRITE_AND_CHECK; val <<= 1; val |= 1; } /* Back at all-ones. */ WRITE_AND_CHECK; /* Move to the right. */ val &= ~x; while (val != -1) { WRITE_AND_CHECK; val >>= 1; val |= x; } break; case 1: /* Initialize everything to 0 and check. */ val = 0; WRITE_AND_CHECK; /* Let the 1 do the dance to the left. */ val = 1; while (val != 0) { WRITE_AND_CHECK; val <<= 1; } WRITE_AND_CHECK; /* And now back to the right again. */ val |= x; while (val != 0) { WRITE_AND_CHECK; val >>= 1; val &= ~x; } break; default: panic("%s: binary is hard: %tu\n", __func__, pat); } #undef WRITE_AND_CHECK /* Restore original value. */ *(uintptr_t *)addr = sav; return (0); } static uint64_t memtest_addrwalk_nop(struct memtest_test *t __unused, vm_paddr_t pa __unused, void *addr __unused, uintptr_t pat __unused) { return (-1); } static uint64_t _memtest_fullpage(struct memtest_test *t, vm_paddr_t pa, void *addr, uintptr_t pat) { uint64_t start, end; uintptr_t val; int i; KASSERT(((uintptr_t)addr & PAGE_MASK) == 0, ("%s: test '%s': %p not " "page aligned", __func__, t->name, addr)); start = rdtsc(); pagefill(addr, pat); if (pagevalidate(addr, pat) != 0) { end = rdtsc(); memtest_log_event(end, "Memtest '%s' FAILURE in page starting " "at 0x%0*jx: doing individual address checks", t->name, (int)sizeof(vm_paddr_t)*2, (uintmax_t)pa); for (i = 0; i < PAGE_SIZE; i += sizeof(pat)) { val = *(volatile uintptr_t *)((uintptr_t)addr + i); if (val != pat) memtest_log_bad_addr(t, pa + i, sizeof(pat), pat, val); } } else { end = rdtsc(); t->bw_iter_ticks += (end - start); t->bw_iter_bytes += 2 * PAGE_SIZE; } return (0); } static uint64_t memtest_fullpage(struct memtest_test *t, vm_paddr_t pa, void *addr, uintptr_t pat) { /* * Save original value(s) at location. */ memtest_data_save(addr, PAGE_SIZE); (void) _memtest_fullpage(t, pa, addr, pat); /* * Restore original value. */ memtest_data_restore(addr, PAGE_SIZE); return (0); } static uint64_t memtest_fullpage_inversion(struct memtest_test *t, vm_paddr_t pa, void *addr, uintptr_t pat) { /* * Save original value(s) at location. */ memtest_data_save(addr, PAGE_SIZE); (void) _memtest_fullpage(t, pa, addr, pat); (void) _memtest_fullpage(t, pa, addr, ~pat); /* * Restore original value. */ memtest_data_restore(addr, PAGE_SIZE); return (0); } static uint64_t memtest_freebsd_classic_logic(struct memtest_test *t, vm_paddr_t pa, void *addr, uintptr_t pat __unused) { int tmp, val, *ptr; ptr = (int *)addr; tmp = *(int *)ptr; /* * Test for alternating 1's and 0's */ *(volatile int *)ptr = 0xaaaaaaaa; val = *(volatile int *)ptr; if (val != 0xaaaaaaaa) memtest_log_bad_addr(t, pa, sizeof(val), 0xaaaaaaaa, val); /* * Test for alternating 0's and 1's */ *(volatile int *)ptr = 0x55555555; val = *(volatile int *)ptr; if (val != 0x55555555) memtest_log_bad_addr(t, pa, sizeof(val), 0x55555555, val); /* * Test for all 1's */ *(volatile int *)ptr = 0xffffffff; val = *(volatile int *)ptr; if (val != 0xffffffff) memtest_log_bad_addr(t, pa, sizeof(val), 0xffffffff, val); /* * Test for all 0's */ *(volatile int *)ptr = 0x0; val = *(volatile int *)ptr; if (val != 0x0) memtest_log_bad_addr(t, pa, sizeof(val), 0x0, val); /* * Restore original value. */ *(int *)ptr = tmp; return (0); } static uint64_t memtest_run_test(struct memtest_test *t, uintptr_t pat) { pt_entry_t *pte; vm_paddr_t end, pa, first; uint64_t start, now; int i, p; #ifdef RASUM_MEMTEST_KERNEL vm_paddr_t rasum_pa_start; int count; #endif if (t == NULL || t->lfunc == NULL) return (-1); start = rdtsc(); pte = CMAP1; /* * physmap is in bytes, so when converting to page boundaries, * round up the start address and round down the end address. */ for (i = 0; i <= memtest_meta.physmap_idx; i += 2) { end = ptoa((vm_paddr_t)Maxmem); if (memtest_meta.physmap[i + 1] < end) end = trunc_page(memtest_meta.physmap[i + 1]); first = round_page(memtest_meta.physmap[i]); #ifdef RASUM_MEMTEST_KERNEL rasum_pa_start = first; count = 0; #endif for (pa = first; pa < end; pa += PAGE_SIZE) { /* * block out kernel memory as not available. */ if (pa >= 0x100000 && pa < memtest_meta.first) continue; /* * block out dcons buffer */ if (memtest_meta.dcons_addr > 0 && pa >= trunc_page(memtest_meta.dcons_addr) && pa < memtest_meta.dcons_addr + memtest_meta.dcons_size) continue; if (t->paselect != NULL && (*t->paselect)(first, end, pa) == 0) continue; /* * map page into kernel: valid, read/write,non-cacheable */ *pte = pa | memtest_get_pg(t->cache_bits); invltlb(); (void) (*t->lfunc)(t, pa, CADDR1, pat); now = rdtsc(); #ifdef RASUM_MEMTEST_KERNEL if ((memtest_meta.flags & ((u_long)1 << MEMTEST_TEST_NO_RASUM)) != 0) goto skip_rasum; /* * Only check and report every 128 pages or at end of * memory region. */ count++; if (count % 128 != 0 && (pa + PAGE_SIZE) < end) goto skip_rasum; getRasumErrors(); struct ecc_summary recc = getEccErrorSummary(); if (recc.cerrs > 0 || recc.ucerrs > 0) { vm_paddr_t rasum_pa_end; rasum_pa_end = (pa + PAGE_SIZE - 1); memtest_log_event(now, "Memtest '%s' FAILURE " "in pages starting at 0x%0*jx ending at " "0x%0*jx: correctable %lu, uncorrectable " "%lu", t->name, (int)sizeof(vm_paddr_t)*2, (uintmax_t)rasum_pa_start, (int)sizeof(vm_paddr_t)*2, (uintmax_t)rasum_pa_end, recc.cerrs, recc.ucerrs); t->err_iter += recc.cerrs; t->err_iter += recc.ucerrs; /* * Zero Rasum summary counters so we do not * repeatedly report errors. */ clearRasumSummaryErrCounters(); /* * We should better cache it upfront or get * it from and print it with rasum above. * Either way we would probably know if start * and end are the same DIMM. */ if ((memtest_meta.flags & ((u_long)1 << MEMTEST_TEST_NO_SMBIOS)) == 0) { memtest_log_smbios_memory_device(t, rasum_pa_start, now); memtest_log_smbios_memory_device(t, rasum_pa_end, now); } } rasum_pa_start = pa + PAGE_SIZE; skip_rasum: #endif /* Check for time limit or user abort. */ if ((pa & 0x4fffff) == 0x400000 && (memtest_timelimit_reached(now) || memtest_user_abort(now))) goto out; /* Intermediate time reporting if taking too long. */ if (((now-start) / memtest_meta.rdtsc_div) >= memtest_meta.interim_dt) { p = atop(pa) * 100 / Maxmem; /* Do a guestimation ignoring memory holes. */ memtest_log_event(now, "Memory test '%s' " "still running (%2d%%)", t->name, p); start = now; } } } out: *pte = 0; invltlb(); return (0); } /* * Returns 0 if we do not want to test that page, 1 if we want to. * For data line tests we will pick one address of that page only. */ static int memtest_dataline_paselect(vm_paddr_t start, vm_paddr_t end, vm_paddr_t pa) { /* Test all start and full end pages of a SMAP region. */ if (pa == start || pa == (end-PAGE_SIZE)) return (1); /* XXX Do a test every 1M for now. */ if (((pa - start) % (256 * PAGE_SIZE)) == 0) return (1); return (0); } /* * Always returns 0 as we do not want to test any addresses in the run * we get called. We just want to build the array of addresses we can test. */ static int memtest_addrwalk_paselect(vm_paddr_t start, vm_paddr_t end, vm_paddr_t pa) { vm_paddr_t addr; int count; /* We are only interested in power-of two addresses. */ addr = pa; count = 0; while (addr > 0) { if (addr & 1) count++; addr >>= 1; } if (count != 1) return (0); memtest_meta.addrwalk_idx++; /* * We want to test as many lines as we can so always add page 0. * For simplicity add the page n times only ignoring the PG_ bits. */ if (memtest_meta.addrwalk_idx == 0) { for (count = sizeof(uintptr_t); count < PAGE_SIZE; count <<= 1) memtest_meta.addrwalk[memtest_meta.addrwalk_idx++] = 0; } KASSERT(memtest_meta.addrwalk_idx < (sizeof(vm_paddr_t) * 8), ("%s: array index out of bounds: %d\n", __func__, memtest_meta.addrwalk_idx)); if (pa != 0) memtest_meta.addrwalk[memtest_meta.addrwalk_idx] = pa; return (0); } /* * "walk0": data line test. */ static uint64_t memtest_walk0(uint64_t begin, struct memtest_test *t) { return (memtest_run_test(t, 0)); } /* * "walk1": data line test. */ static uint64_t memtest_walk1(uint64_t begin, struct memtest_test *t) { return (memtest_run_test(t, 1)); } /* * Address line test walk a single high line. * XXX this is currently entirely unoptimized. The fact that we can only * map a page at a time complicates matters. * XXX Given we have holes we cannot check, and are doing page level * currently, we are missing some lines. */ static uint64_t memtest_addrwalk(uint64_t begin, struct memtest_test *t) { pt_entry_t *pte; vm_paddr_t pa; uintptr_t k, pat, *sav; uint64_t now; int err, i, j; /* * If the address walk array was not yet populated, do so. We assume * the memory layout will never change during the test. */ if (memtest_meta.addrwalk_idx < 0) (void) memtest_run_test(t, -1); if (memtest_meta.addrwalk_idx < 0) { now = rdtsc(); memtest_log_event(now, "Memtest '%s' FAILURE: no address to " "test", t->name); return (-1); } #if defined(__amd64__) #define FILL_PAT 0xaaaaaaaaaaaaaaaa #define FILL_INVPAT 0x5555555555555555 #elif defined(__i386__) #define FILL_PAT 0xaaaaaaaa #define FILL_INVPAT 0x55555555 #else #error Unsupported Architecture #endif pte = CMAP1; /* Save the data. */ sav = (uintptr_t *)memtest_fullpage_save; for (i = 0; i <= memtest_meta.addrwalk_idx; i++) { pa = memtest_meta.addrwalk[i]; /* * map page into kernel: valid, read/write,non-cacheable */ *pte = pa | memtest_get_pg(t->cache_bits); invltlb(); if (pa != 0) { *(sav++) = *(uintptr_t *)CADDR1; continue; } /* First page needs special treatment. */ k = sizeof(uintptr_t) << i; *(sav++) = *(uintptr_t *)((uintptr_t)CADDR1 | k); } err = 1; /* Now do the walks. */ for (i = 0; i <= memtest_meta.addrwalk_idx; i++) { /* * Initialize all addresses with a pattern. Need to do it for * each iteration in case there was an error. Otherwise we will * continue to see the error for all remaining tests while with * this we weill at least have a chance. */ for (j = 0; err != 0 && j <= memtest_meta.addrwalk_idx; j++) { pa = memtest_meta.addrwalk[j]; /* * map page into kernel: valid, read/write,non-cacheable */ *pte = pa | memtest_get_pg(t->cache_bits); invltlb(); if (pa != 0) { *(volatile uintptr_t *)CADDR1 = FILL_PAT; pat = *(volatile uintptr_t *)CADDR1; } else { k = sizeof(uintptr_t) << j; *(volatile uintptr_t *)((uintptr_t)CADDR1 | k) = FILL_PAT; pat = *(volatile uintptr_t *) ((uintptr_t)CADDR1 | k); pa |= k; } if (pat != FILL_PAT) memtest_log_bad_addr(t, pa, sizeof(uintptr_t), FILL_PAT, pat); } err = 0; pa = memtest_meta.addrwalk[i]; /* * map page into kernel: valid, read/write,non-cacheable */ *pte = pa | memtest_get_pg(t->cache_bits); invltlb(); if (pa != 0) { *(volatile uintptr_t *)CADDR1 = FILL_INVPAT; } else { k = sizeof(uintptr_t) << i; *(volatile uintptr_t *)((uintptr_t)CADDR1 | k) = FILL_INVPAT; } for (j = 0; j <= memtest_meta.addrwalk_idx; j++) { /* Skip our changed address, will check at end. */ if (j == i) continue; pa = memtest_meta.addrwalk[j]; /* * map page into kernel: valid, read/write,non-cacheable */ *pte = pa | memtest_get_pg(t->cache_bits); invltlb(); if (pa != 0) { pat = *(volatile uintptr_t *)CADDR1; } else { k = sizeof(uintptr_t) << j; pat = *(volatile uintptr_t *) ((uintptr_t)CADDR1 | k); pa |= k; } if (pat != FILL_PAT) { err++; memtest_log_bad_addr(t, pa, sizeof(uintptr_t), FILL_PAT, pat); } } pa = memtest_meta.addrwalk[i]; /* * map page into kernel: valid, read/write,non-cacheable */ *pte = pa | memtest_get_pg(t->cache_bits); invltlb(); if (pa != 0) { pat = *(volatile uintptr_t *)CADDR1; } else { k = sizeof(uintptr_t) << i; pat = *(volatile uintptr_t *)((uintptr_t)CADDR1 | k); pa |= k; } if (pat != FILL_INVPAT) { err++; memtest_log_bad_addr(t, pa, sizeof(uintptr_t), FILL_INVPAT, pat); } /* Clean the data pattern. In case of error we'll swipe all. */ if (err == 0) { /* Re-map to avoid problems when on page 0. */ pa = memtest_meta.addrwalk[i]; /* * map page into kernel: valid, read/write,non-cacheable */ *pte = pa | memtest_get_pg(t->cache_bits); invltlb(); if (pa != 0) { *(volatile uintptr_t *)CADDR1 = FILL_PAT; } else { k = sizeof(uintptr_t) << i; *(volatile uintptr_t *)((uintptr_t)CADDR1 | k) = FILL_PAT; } } } #undef FILL_PAT #undef FILL_INVPAT /* Restore the original data. */ sav = (uintptr_t *)memtest_fullpage_save; for (i = 0; i <= memtest_meta.addrwalk_idx; i++) { pa = memtest_meta.addrwalk[i]; /* * map page into kernel: valid, read/write,non-cacheable */ *pte = pa | memtest_get_pg(t->cache_bits); invltlb(); if (pa != 0) { *(uintptr_t *)CADDR1 = *(sav++); continue; } /* First page needs special treatment. */ k = sizeof(uintptr_t) << i; *(uintptr_t *)((uintptr_t)CADDR1 | k) = *(sav++); } *pte = 0; invltlb(); return (0); } static uint64_t memtest_allones(uint64_t begin, struct memtest_test *t) { return (memtest_run_test(t, -1)); } static uint64_t memtest_allzeros(uint64_t begin, struct memtest_test *t) { return (memtest_run_test(t, 0x0)); } static uint64_t memtest_userpattern(uint64_t begin, struct memtest_test *t) { return (memtest_run_test(t, memtest_meta.upattern)); } #if defined(__amd64__) static uint64_t memtest_inversion(uint64_t begin, struct memtest_test *t) { return (memtest_run_test(t, 0x1020304050607080)); } #elif defined(__i386__) static uint64_t memtest_inversion(uint64_t begin, struct memtest_test *t) { return (memtest_run_test(t, 0x10203040)); } #else #error Unsupported architecture #endif static uint64_t memtest_freebsd_classic(uint64_t begin, struct memtest_test *t) { return (memtest_run_test(t, -1)); } static struct memtest_test memtest_all_tests[MEMTEST_TEST_MAX_SHIFT+1] = { [MEMTEST_TEST_WALK0] = { .name = "walk0", .descr = "walking low data line test", .cache_bits = MEMTEST_PG_NOCACHE, .tfunc = memtest_walk0, .lfunc = memtest_dataline_walk, .paselect = memtest_dataline_paselect, }, [MEMTEST_TEST_WALK1] = { .name = "walk1", .descr = "walking high data line test", .cache_bits = MEMTEST_PG_NOCACHE, .tfunc = memtest_walk1, .lfunc = memtest_dataline_walk, .paselect = memtest_dataline_paselect, }, [MEMTEST_TEST_ADDRWALK] = { .name = "addrwalk", .descr = "walking high address line test", .cache_bits = MEMTEST_PG_NOCACHE, .tfunc = memtest_addrwalk, .lfunc = memtest_addrwalk_nop, .paselect = memtest_addrwalk_paselect, }, [MEMTEST_TEST_ALLONES] = { .name = "allones", .descr = "fill memory with all-ones and validate", .cache_bits = MEMTEST_PG_PAGEFILL, .tfunc = memtest_allones, .lfunc = memtest_fullpage, }, [MEMTEST_TEST_ALLZEROS] = { .name = "allzeros", .descr = "fill memory with all-zeros and validate", .cache_bits = MEMTEST_PG_PAGEFILL, .tfunc = memtest_allzeros, .lfunc = memtest_fullpage, }, [MEMTEST_TEST_USERPATTERN] = { .name = "userpattern", .descr = "fill memory with user pattern and validate", .cache_bits = MEMTEST_PG_PAGEFILL, .tfunc = memtest_userpattern, .lfunc = memtest_fullpage, }, [MEMTEST_TEST_INVERSION] = { .name = "inversion", .descr = "fill memory with 0x1020304050607080, " "validate, write inverted, validate", .cache_bits = MEMTEST_PG_PAGEFILL, .tfunc = memtest_inversion, .lfunc = memtest_fullpage_inversion, }, [MEMTEST_TEST_FREEBSD_CLASSIC] = { .name = "FreeBSD classic", .descr = "classic FreeBSD pattern tests on beginning " "of each page", .cache_bits = MEMTEST_PG_NOCACHE, .tfunc = memtest_freebsd_classic, .lfunc = memtest_freebsd_classic_logic, }, }; static void memtest_log_start(const char *fmt) { printf("[%6lu] %s\n", 0lu, fmt); } static void memtest_log_end(const char *fmt) { printf("[%6ju] %s\n", (memtest_meta.end - memtest_meta.start) / memtest_meta.rdtsc_div, fmt); } static void memtest_print_bw(uint64_t ticks, uint64_t bytes, char *buf, size_t len) { const char *m; uint64_t bw; if (buf == NULL || len == 0) return; if (ticks == 0 || bytes == 0) { buf[0] = '\0'; return; } m = ""; if (ticks > memtest_meta.rdtsc_div) { bw = bytes; bw /= (ticks / memtest_meta.rdtsc_div); } else { bw = bytes * memtest_meta.rdtsc_div / ticks; } if (bw > 1000) bw /= 1000, m = "K"; if (bw > 1000) bw /= 1000, m = "M"; if (bw > 1000) bw /= 1000, m = "G"; if (bw > 1000) bw /= 1000, m = "T"; snprintf(buf, len, " (%ju %sB/s)", (uintmax_t)bw, m); #ifdef MEMTEST_DEBUG uint64_t now = rdtsc(); memtest_log_event(now, "bytes = %ju, ticks = %ju, div = %ju, %s", (uintmax_t)bytes, (uintmax_t)ticks, (uintmax_t)memtest_meta.rdtsc_div, buf); #endif } static void memtest_log_test_summary(uint64_t end, struct memtest_test *t) { char buf[16]; t->iter++; t->bw_total_ticks += t->bw_iter_ticks; t->bw_total_bytes += t->bw_iter_bytes; if (t->err_iter) { t->err_total += t->err_iter; t->err_count++; } buf[0] = '\0'; memtest_print_bw(t->bw_iter_ticks, t->bw_iter_bytes, buf, sizeof(buf)); memtest_log_event(end, "Summary for '%s': %ju failures on iteration %lu: " "%ju PASS, %ju FAIL%s", t->name, t->err_iter, t->iter, (t->iter - t->err_count), t->err_count, buf); } static void memtest_log_iter_summary(uint64_t end, u_long iter) { struct memtest_test *t; char buf[16]; uint64_t bw_b, bw_t, errs, tfailed, trun; u_long memtest_idx; bw_b = bw_t = errs = tfailed = trun = 0; for (memtest_idx = 0; memtest_idx <= MEMTEST_TEST_MAX_SHIFT; memtest_idx++) { if ((memtest_meta.tests & ((u_long)1 << memtest_idx)) == 0) continue; t = &memtest_all_tests[memtest_idx]; if (t == NULL) continue; /* Run this iteration? */ if (t->iter != iter) continue; trun++; if (t->err_iter > 0) { tfailed++; errs += t->err_iter; } /* Not all tests run might have bandwidth statistics. */ if (t->bw_iter_ticks > 0) { bw_b += t->bw_iter_bytes; bw_t += t->bw_iter_ticks; } } buf[0] = '\0'; memtest_print_bw(bw_t, bw_b, buf, sizeof(buf)); memtest_log_event(end, "This iteration: %ju PASS, %ju FAIL, %ju new errors%s", (uintmax_t)(trun - tfailed), (uintmax_t)tfailed, (uintmax_t)errs, buf); } static const char * memtest_log_early_end(void) { if (memtest_meta.completion_mode & MEMTEST_COMPLETION_TIMEOUT) return (" Last iteration ended early due to set time limit"); else if (memtest_meta.completion_mode & MEMTEST_COMPLETION_USER_ABORT) return (" Last iteration ended early due to user abort"); return (""); } static void memtest_log_total_summary(void) { struct memtest_test *t; char buf[16]; uint64_t bw_t, bw_b, errs, tfailed, trun; u_long memtest_idx; memtest_log_event(memtest_meta.end, "Done %lu iteration%s.%s", memtest_meta.iter, (memtest_meta.iter == 1) ? "" : "s", memtest_log_early_end()); bw_b = bw_t = errs = tfailed = trun = 0; for (memtest_idx = 0; memtest_idx <= MEMTEST_TEST_MAX_SHIFT; memtest_idx++) { if ((memtest_meta.tests & ((u_long)1 << memtest_idx)) == 0) continue; t = &memtest_all_tests[memtest_idx]; if (t == NULL) continue; trun += t->iter; if (t->err_total > 0) { tfailed++; errs += t->err_total; } /* Not all tests run might have bandwidth statistics. */ if (t->bw_total_ticks > 0) { bw_b += t->bw_total_bytes; bw_t += t->bw_total_ticks; } buf[0] = '\0'; memtest_print_bw(t->bw_total_ticks, t->bw_total_bytes, buf, sizeof(buf)); memtest_log_event(memtest_meta.end, "Test summary for '%s': %ju PASS, %ju FAIL%s", t->name, (uintmax_t)(t->iter - t->err_count), (uintmax_t)t->err_count, buf); } buf[0] = '\0'; memtest_print_bw(bw_t, bw_b, buf, sizeof(buf)); memtest_log_event(memtest_meta.end, "Summary: %ju PASS, %ju FAIL, %ju total errors%s", (uintmax_t)(trun - tfailed), (uintmax_t)tfailed, (uintmax_t)errs, buf); } SYSCTL_NODE(_machdep, OID_AUTO, memtest, CTLFLAG_RD, 0, "Memory testing"); static int sysctl_memtest_summary(SYSCTL_HANDLER_ARGS) { struct memtest_test *t; char buf[16]; struct sbuf sb; uint64_t bw_t, bw_b, errs, tfailed, trun; u_long memtest_idx; int error; sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND); sbuf_printf(&sb, "\n[%6ju] Done %lu iteration%s.%s\n", ((memtest_meta.end - memtest_meta.start) / memtest_meta.rdtsc_div), memtest_meta.iter, (memtest_meta.iter == 1) ? "" : "s", memtest_log_early_end()); bw_b = bw_t = errs = tfailed = trun = 0; for (memtest_idx = 0; memtest_idx <= MEMTEST_TEST_MAX_SHIFT; memtest_idx++) { if ((memtest_meta.tests & ((u_long)1 << memtest_idx)) == 0) continue; t = &memtest_all_tests[memtest_idx]; if (t == NULL) continue; trun += t->iter; if (t->err_total > 0) { tfailed++; errs += t->err_total; } /* Not all tests run might have bandwidth statistics. */ if (t->bw_total_ticks > 0) { bw_b += t->bw_total_bytes; bw_t += t->bw_total_ticks; } buf[0] = '\0'; memtest_print_bw(t->bw_total_ticks, t->bw_total_bytes, buf, sizeof(buf)); sbuf_printf(&sb, "[%6ju] " "Test summary for '%s': %ju PASS, %ju FAIL%s\n", ((memtest_meta.end - memtest_meta.start) / memtest_meta.rdtsc_div), t->name, (uintmax_t)(t->iter - t->err_count), (uintmax_t)t->err_count, buf); } buf[0] = '\0'; memtest_print_bw(bw_t, bw_b, buf, sizeof(buf)); sbuf_printf(&sb, "[%6ju] " "Summary: %ju PASS, %ju FAIL, %ju total errors%s\n", ((memtest_meta.end - memtest_meta.start) / memtest_meta.rdtsc_div), (uintmax_t)(trun - tfailed), (uintmax_t)tfailed, (uintmax_t)errs, buf); sbuf_trim(&sb); sbuf_finish(&sb); error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); sbuf_delete(&sb); return (error); } SYSCTL_PROC(_machdep_memtest, OID_AUTO, summary, CTLTYPE_STRING | CTLFLAG_RD, 0, 0, sysctl_memtest_summary, "A", "Summary of memory tests"); static int sysctl_memtest_bw_total(SYSCTL_HANDLER_ARGS) { struct memtest_test *t; struct sbuf sb; char buf[16]; int error; sbuf_new(&sb, NULL, 128, SBUF_AUTOEXTEND); t = (struct memtest_test *)oidp->oid_arg1; if (t != NULL) { buf[0] = '\0'; memtest_print_bw(t->bw_total_ticks, t->bw_total_bytes, buf, sizeof(buf)); sbuf_printf(&sb, "%s", buf); } else { sbuf_printf(&sb, "N/A"); } sbuf_trim(&sb); sbuf_finish(&sb); error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); sbuf_delete(&sb); return (error); } static void memtest_add_sysctl(void) { struct sysctl_oid *oid; struct memtest_test *t; u_long memtest_idx; /* We will not use a context as we have no intend to ever free it. */ for (memtest_idx = 0; memtest_idx <= MEMTEST_TEST_MAX_SHIFT; memtest_idx++) { if ((memtest_meta.tests & ((u_long)1 << memtest_idx)) == 0) continue; t = &memtest_all_tests[memtest_idx]; if (t == NULL) continue; oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_machdep_memtest), OID_AUTO, t->name, CTLFLAG_RW, NULL, t->descr); if (oid == NULL) continue; (void) SYSCTL_ADD_ULONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "iter", CTLFLAG_RD|CTLTYPE_ULONG, &t->iter, "Number of iterations this test ran"); (void) SYSCTL_ADD_ULONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "err_count", CTLFLAG_RD|CTLTYPE_ULONG, &t->err_count, "Number of iterations with errors"); (void) SYSCTL_ADD_ULONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "err_total", CTLFLAG_RD|CTLTYPE_ULONG, &t->err_total, "Total number of errors"); (void) SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "bw_total", CTLFLAG_RD|CTLTYPE_STRING, t, sizeof(t), sysctl_memtest_bw_total, "A", "Average bandwidth over all iterations"); } } /* For the lack of a better subsystem use something after SI_SUB_KMEM */ SYSINIT(memtest_sysctl, SI_SUB_VM_CONF, SI_ORDER_ANY, memtest_add_sysctl, 0); void getmemsize2(uintptr_t first, vm_paddr_t *physmap, int physmap_idx, int has_smap) { quad_t dcons_addr, dcons_size; uint64_t now; vm_paddr_t pa; u_long physmem_tunable, tmpul; int off, i, pa_indx, da_indx, memtest_idx; /* * Initialize in-kernel memory testing. */ memtest_meta.flags = 0; if (TUNABLE_ULONG_FETCH("hw.memtest.verbose", &tmpul)) { memtest_meta.flags = tmpul & MEMTEST_TEST_ALL_MASK; memtest_meta.flags &= ~((u_long)1 << MEMTEST_TEST_NO_SAVE); } TUNABLE_ULONG_FETCH("hw.memtest.loops", &memtest_loops); memtest_meta.interim_dt = MEMTEST_INTERIM_REPORT_DT; if (TUNABLE_ULONG_FETCH("hw.memtest.interim_report_dt", &tmpul)) memtest_meta.interim_dt = tmpul; memtest_meta.completion_mode = MEMTEST_COMPLETION_BOOT; if (TUNABLE_ULONG_FETCH("hw.memtest.completion_mode", &tmpul)) { if (tmpul >= MEMTEST_COMPLETION_BOOT && tmpul <= MEMTEST_COMPLETION_WAIT) memtest_meta.completion_mode = tmpul; else printf("hw.memtest.completion_mode=%lu out of range, " "using default %lu\n", tmpul, memtest_meta.completion_mode); } memtest_meta.time_limit = 0; if (TUNABLE_ULONG_FETCH("hw.memtest.time_limit", &tmpul)) memtest_meta.time_limit = tmpul; memtest_meta.tests = 0; if (TUNABLE_ULONG_FETCH("hw.memtest.tests", &tmpul)) { if (tmpul & ~MEMTEST_TEST_ALL_MASK) { memtest_meta.tests = tmpul & MEMTEST_TEST_ALL_MASK; printf("hw.memtest.test=0x%016lx asks for unknown " "tests, masking to valid 0x%016lx\n", tmpul, memtest_meta.tests); } else { memtest_meta.tests = tmpul; } } /* Check if we should never save/restore pages we run tests on. */ if (memtest_meta.tests & ((u_long)1 << MEMTEST_TEST_NO_SAVE)) { memtest_meta.flags |= ((u_long)1 << MEMTEST_TEST_NO_SAVE); memtest_meta.tests &= ~((u_long)1 << MEMTEST_TEST_NO_SAVE); } /* Check if we should not try to probe and use SMBIOS information. */ if (memtest_meta.tests & ((u_long)1 << MEMTEST_TEST_NO_SMBIOS)) { memtest_meta.flags |= ((u_long)1 << MEMTEST_TEST_NO_SMBIOS); memtest_meta.tests &= ~((u_long)1 << MEMTEST_TEST_NO_SMBIOS); } #ifdef RASUM_MEMTEST_KERNEL /* Check if we should not try to use RASUM information. */ if (memtest_meta.tests & ((u_long)1 << MEMTEST_TEST_NO_RASUM)) { memtest_meta.flags |= ((u_long)1 << MEMTEST_TEST_NO_RASUM); memtest_meta.tests &= ~((u_long)1 << MEMTEST_TEST_NO_RASUM); } #endif memtest_meta.upattern = 0;; if (TUNABLE_ULONG_FETCH("hw.memtest.user_pattern", &tmpul)) { memtest_meta.upattern = tmpul; } else { if (memtest_meta.tests & ((u_long)1 << MEMTEST_TEST_USERPATTERN)) { printf("hw.memtest.user_pattern not set but " "user pattern test requested. Disabling\n"); memtest_meta.tests &= ~((u_long)1 << MEMTEST_TEST_USERPATTERN); } } /* * Maxmem isn't the "maximum memory", it's one larger than the * highest page of the physical address space. It should be * called something like "Maxphyspage". We may adjust this * based on ``hw.physmem'' and the results of the memory test. */ Maxmem = atop(physmap[physmap_idx + 1]); #ifdef MAXMEM Maxmem = MAXMEM / 4; #endif if (TUNABLE_ULONG_FETCH("hw.physmem", &physmem_tunable)) { Maxmem = atop(physmem_tunable); if (memtest_meta.tests != 0 && (boothowto & RB_VERBOSE) == 0) printf("Memory tests limited to 0x%jx\n", (uintmax_t)Maxmem * 4); } /* * If we have an SMAP, don't allow MAXMEM or hw.physmem to extend * the amount of memory in the system. */ if (has_smap && Maxmem > atop(physmap[physmap_idx + 1])) Maxmem = atop(physmap[physmap_idx + 1]); #if defined(__i386__) /* * If Maxmem has been increased beyond what the system has detected, * extend the last memory segment to the new limit. */ if (atop(physmap[physmap_idx + 1]) < Maxmem) physmap[physmap_idx + 1] = ptoa((vm_paddr_t)Maxmem); #endif if (atop(physmap[physmap_idx + 1]) != Maxmem && (boothowto & RB_VERBOSE)) printf("Physical memory use set to %ldK\n", Maxmem * 4); /* call pmap initialization to make new kernel address space */ pmap_bootstrap(&first); /* * Size up each available chunk of physical memory. */ physmap[0] = PAGE_SIZE; /* mask off page 0 */ pa_indx = 0; da_indx = 1; phys_avail[pa_indx++] = physmap[0]; phys_avail[pa_indx] = physmap[0]; dump_avail[da_indx] = physmap[0]; /* * Get dcons buffer address */ if (getenv_quad("dcons.addr", &dcons_addr) == 0 || getenv_quad("dcons.size", &dcons_size) == 0) dcons_addr = 0; #ifndef XEN /* * Run memory test(s). */ if (memtest_meta.tests != 0) { register_t reg; /* * Fill in all the meta data once, so all tests can access it * read-only. */ memtest_meta.physmap_idx = physmap_idx; memtest_meta.physmap = physmap; memtest_meta.first = first; memtest_meta.dcons_addr = dcons_addr; memtest_meta.dcons_size = dcons_size; /* * We are still before cpu_est_clockrate() so do it ourself but * we can simplify give we are basically UP still. */ reg = intr_disable(); memtest_meta.start = rdtsc(); DELAY(1000); now = rdtsc(); intr_restore(reg); memtest_meta.rdtsc_div = now - memtest_meta.start; memtest_meta.rdtsc_div = memtest_meta.rdtsc_div * 1000 - memtest_meta.rdtsc_div * 5; if ((memtest_meta.flags & ((u_long)1 << MEMTEST_TEST_NO_SMBIOS)) == 0) memtest_find_smbios(); #ifdef RASUM_MEMTEST_KERNEL if ((memtest_meta.flags & ((u_long)1 << MEMTEST_TEST_NO_RASUM)) == 0) if (probeRasum() != 0) { printf("Rasum probing returned error, " "disabling\n"); memtest_meta.flags |= ((u_long)1 << MEMTEST_TEST_NO_RASUM); } #endif memtest_meta.completion_mode &= ~MEMTEST_COMPLETION_END_MASK; memtest_meta.start = now = rdtsc(); memtest_log_start("Memory tests starting"); memtest_log_start("Press CTRL-c to abort"); for (memtest_meta.iter = 0; memtest_meta.iter < memtest_loops; memtest_meta.iter++) { now = rdtsc(); memtest_log_event(now, "Memory test iteration %lu of " "%lu starting", memtest_meta.iter+1, memtest_loops); /* * Reset statistics for this test iteration. We need * to do it upfront in case we will break the work loop * hitting a timelimit set. */ for (memtest_idx = 0; memtest_idx <= MEMTEST_TEST_MAX_SHIFT; memtest_idx++) { struct memtest_test *t; if ((memtest_meta.tests & ((u_long)1 << memtest_idx)) == 0) continue; t = &memtest_all_tests[memtest_idx]; if (t == NULL) continue; t->err_iter = 0; t->bw_iter_ticks = t->bw_iter_bytes = 0; } for (memtest_idx = 0; memtest_idx <= MEMTEST_TEST_MAX_SHIFT; memtest_idx++) { struct memtest_test *t; if ((memtest_meta.tests & ((u_long)1 << memtest_idx)) == 0) continue; /* Skip tests not implemented. */ t = &memtest_all_tests[memtest_idx]; if (t == NULL) continue; if (t->tfunc == NULL) continue; now = rdtsc(); memtest_log_event(now, "Memory test '%s' " "starting (%s)", t->name, t->descr); /* Run this memory test. */ (void) (*t->tfunc)(now, t); now = rdtsc(); memtest_log_event(now, "Memory test '%s' " "completed", t->name); memtest_log_test_summary(now, t); if (memtest_timelimit_reached(now) || memtest_user_abort(now)) break; } now = rdtsc(); memtest_log_event(now, "Memory test iteration %lu of " "%lu completed", memtest_meta.iter+1, memtest_loops); memtest_log_iter_summary(now, memtest_meta.iter+1); if (memtest_meta.completion_mode & MEMTEST_COMPLETION_END_MASK) break; } memtest_meta.end = rdtsc(); memtest_log_end("Memory tests completed"); memtest_log_total_summary(); switch (memtest_meta.completion_mode) { case MEMTEST_COMPLETION_WAIT: printf("The operating system has halted\n"); printf("Please manually reset machine\n"); cpu_halt(); /* NOTREACHED */ break; case MEMTEST_COMPLETION_REBOOT: printf("Rebooting\n"); /* Give the printf a chance ... even if short. */ DELAY(1000000); cpu_reset(); break; case MEMTEST_COMPLETION_BOOT: default: /* Just in case play save. */ break; } } /* * physmap is in bytes, so when converting to page boundaries, * round up the start address and round down the end address. */ for (i = 0; i <= physmap_idx; i += 2) { vm_paddr_t end; end = ptoa((vm_paddr_t)Maxmem); if (physmap[i + 1] < end) end = trunc_page(physmap[i + 1]); for (pa = round_page(physmap[i]); pa < end; pa += PAGE_SIZE) { int page_bad, full, j; full = FALSE; /* * block out kernel memory as not available. */ #if defined(__amd64__) if (pa >= 0x100000 && pa < first) #elif defined(__i386__) if (pa >= KERNLOAD && pa < first) #endif goto do_dump_avail; /* * block out dcons buffer */ if (dcons_addr > 0 && pa >= trunc_page(dcons_addr) && pa < dcons_addr + dcons_size) goto do_dump_avail; page_bad = FALSE; /* Check for faulty pages. */ if (memtest_meta.physfaulty_idx >= PHYS_ERR_ARRAY_END && pa >= phys_faulty[memtest_meta.physfaulty_idx - 1]) page_bad = TRUE; else for (j = 0; j < memtest_meta.physfaulty_idx; j += 2) { if (pa >= phys_faulty[j+1]) break; if (pa >= phys_faulty[j] && pa < phys_faulty[j+1]) { page_bad = TRUE; break; } } /* * Adjust array of valid/good pages. */ if (page_bad == TRUE) continue; /* * If this good page is a continuation of the * previous set of good pages, then just increase * the end pointer. Otherwise start a new chunk. * Note that "end" points one higher than end, * making the range >= start and < end. * If we're also doing a speculative memory * test and we at or past the end, bump up Maxmem * so that we keep going. The first bad page * will terminate the loop. */ if (phys_avail[pa_indx] == pa) { phys_avail[pa_indx] += PAGE_SIZE; } else { pa_indx++; if (pa_indx == PHYS_AVAIL_ARRAY_END) { printf( "Too many holes in the physical address space, giving up\n"); pa_indx--; full = TRUE; goto do_dump_avail; } phys_avail[pa_indx++] = pa; /* start */ phys_avail[pa_indx] = pa + PAGE_SIZE; /* end */ } physmem++; do_dump_avail: if (dump_avail[da_indx] == pa) { dump_avail[da_indx] += PAGE_SIZE; } else { da_indx++; if (da_indx == DUMP_AVAIL_ARRAY_END) { da_indx--; goto do_next; } dump_avail[da_indx++] = pa; /* start */ dump_avail[da_indx] = pa + PAGE_SIZE; /* end */ } do_next: if (full) break; } } #else /* XEN */ phys_avail[0] = physfree; phys_avail[1] = xen_start_info->nr_pages*PAGE_SIZE; dump_avail[0] = 0; dump_avail[1] = xen_start_info->nr_pages*PAGE_SIZE; #endif /* !XEN */ /* * XXX * The last chunk must contain at least one page plus the message * buffer to avoid complicating other code (message buffer address * calculation, etc.). */ while (phys_avail[pa_indx - 1] + PAGE_SIZE + round_page(MSGBUF_SIZE) >= phys_avail[pa_indx]) { physmem -= atop(phys_avail[pa_indx] - phys_avail[pa_indx - 1]); phys_avail[pa_indx--] = 0; phys_avail[pa_indx--] = 0; } Maxmem = atop(phys_avail[pa_indx]); /* Trim off space for the message buffer. */ phys_avail[pa_indx] -= round_page(MSGBUF_SIZE); /* Map the message buffer. */ for (off = 0; off < round_page(MSGBUF_SIZE); off += PAGE_SIZE) pmap_kenter((vm_offset_t)msgbufp + off, phys_avail[pa_indx] + off); }