[PATCH v3 0/2] IPMI watchdog support

[Henning Schild]

changes since v2:
- rebased on next (dropped former p1)
- added 5s timeout guard around IPMI cmds
- return EFI_DEVICE_ERROR from init when retry loop times out
- introduced Stall call in wait function
- add a way to only probe once, IPMI is not PCI bound and once is
enough

changes since v1:
- whitespace fix
- remove temporary editor file from p3

These patches add support for IPMI watchdogs typically found in
server-class hardware. Machines having IPMI will in any version have a
watchdog as well.

The Linux iTCO driver has special vendor logic for Supermicro, where the
iTCO does not work in favour of IPMI. So we place that one before itco
like done with the IPC 427E.

This driver has error handling which was kind of hard to test, but the
code can be easily placed into a user-space application with iopl(3) and
"no action". Where ipmitool and other helpers can be used.
The error handling also means we could end up in the unlikely and
unfortunate situation where we would have to boot without the watchdog
being actually armed. For testing i modified the code to cause some
errors, and the retries and recovery worked. So i hope it would be
highly unlikely that we fail to arm.


Henning Schild (2):
Move smbios helper function into utils.
watchdog: ipmi: Add driver for machines with IPMI

Makefile.am | 2 +
drivers/watchdog/ipc4x7e_wdt.c | 25 ----
drivers/watchdog/ipmi_wdt.c | 209 +++++++++++++++++++++++++++++++++
include/utils.h | 3 +
utils.c | 25 ++++
5 files changed, 239 insertions(+), 25 deletions(-)
create mode 100644 drivers/watchdog/ipmi_wdt.c

--
2.39.3

[Henning Schild]

The code can also prove useful for other use-cases where smbios
information needs to be accessed. Move it to a central place to allow
for such future users.

Signed-off-by: Henning Schild <henning...@siemens.com>
---
drivers/watchdog/ipc4x7e_wdt.c | 25 -------------------------
include/utils.h | 3 +++
utils.c | 25 +++++++++++++++++++++++++
3 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/drivers/watchdog/ipc4x7e_wdt.c b/drivers/watchdog/ipc4x7e_wdt.c
index 6a5c0f0f4632..f7e5e6a2ef9b 100644
--- a/drivers/watchdog/ipc4x7e_wdt.c
+++ b/drivers/watchdog/ipc4x7e_wdt.c
@@ -80,31 +80,6 @@ static UINT32 get_station_id(SMBIOS_STRUCTURE_POINTER oem_strct)
return 0;
}

-static SMBIOS_STRUCTURE_POINTER
-smbios_find_struct(SMBIOS_STRUCTURE_TABLE *table, UINT16 type)
-{
- SMBIOS_STRUCTURE_POINTER strct;
- UINT8 *str;
- UINTN n;
-
- strct.Raw = (UINT8 *)(uintptr_t)table->TableAddress;
-
- for (n = 0; n < table->NumberOfSmbiosStructures; n++) {
- if (strct.Hdr->Type == type) {
- return strct;
- }
- /* Read over any appended strings. */
- str = strct.Raw + strct.Hdr->Length;
- while (str[0] != 0 || str[1] != 0) {
- str++;
- }
- strct.Raw = str + 2;
- }
-
- strct.Raw = NULL;
- return strct;
-}
-
static UINTN mmcfg_address(UINTN bus, UINTN device, UINTN function,
UINTN offset)
{
diff --git a/include/utils.h b/include/utils.h
index 084796e23222..15d60fbdbc66 100644
--- a/include/utils.h
+++ b/include/utils.h
@@ -56,3 +56,6 @@ VOID PrintC(const UINT8 color, const CHAR16 *fmt, ...);

#define INFO(fmt, ...) \
PrintC(EFI_LIGHTGRAY, fmt, ##__VA_ARGS__)
+
+SMBIOS_STRUCTURE_POINTER smbios_find_struct(SMBIOS_STRUCTURE_TABLE *table,
+ UINT16 type);
diff --git a/utils.c b/utils.c
index 0ae5a5e0b8dc..e065b6f5d800 100644
--- a/utils.c
+++ b/utils.c
@@ -295,3 +295,28 @@ CHAR16 *GetBootMediumPath(CHAR16 *input)

return dst;
}
+
+SMBIOS_STRUCTURE_POINTER smbios_find_struct(SMBIOS_STRUCTURE_TABLE *table,
+ UINT16 type)
+{
+ SMBIOS_STRUCTURE_POINTER strct;
+ UINT8 *str;
+ UINTN n;
+
+ strct.Raw = (UINT8 *)(uintptr_t)table->TableAddress;
+
+ for (n = 0; n < table->NumberOfSmbiosStructures; n++) {
+ if (strct.Hdr->Type == type) {
+ return strct;
+ }
+ /* Read over any appended strings. */
+ str = strct.Raw + strct.Hdr->Length;
+ while (str[0] != 0 || str[1] != 0) {
+ str++;
+ }
+ strct.Raw = str + 2;
+ }
+
+ strct.Raw = NULL;
+ return strct;
+}
--
2.39.3

[Henning Schild]

Tested on a Supermicro X11 series (Simatic IPC 1047E) and on some other
random Supermicro X9 series machine. Should work for any board that has
IPMI in any version. When an IPMI watchdog is present, iTCO is likely
not working. The Linux driver would detect that and has special code to
deal with especially Supermicro, here we use probe order.

Developed using the IPMI Spec 2.0 sections 9 and 27.

Signed-off-by: Henning Schild <henning...@siemens.com>
---

Makefile.am | 2 +
drivers/watchdog/ipmi_wdt.c | 209 ++++++++++++++++++++++++++++++++++++
2 files changed, 211 insertions(+)
create mode 100644 drivers/watchdog/ipmi_wdt.c

diff --git a/Makefile.am b/Makefile.am
index 48c560f72f0c..ba2440025a0c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -159,12 +159,14 @@ if BOOTLOADER
if ARCH_IS_X86
# NOTE: wdat.c is placed first so it is tried before any other drivers
# NOTE: ipc4x7e_wdt.c must be *before* itco.c
+# NOTE: ipmi_wdt.c must be *before* itco.c
efi_sources_watchdogs = \
drivers/watchdog/wdat.c \
drivers/watchdog/amdfch_wdt.c \
drivers/watchdog/i6300esb.c \
drivers/watchdog/atom-quark.c \
drivers/watchdog/ipc4x7e_wdt.c \
+ drivers/watchdog/ipmi_wdt.c \
drivers/watchdog/itco.c \
drivers/watchdog/hpwdt.c
else
diff --git a/drivers/watchdog/ipmi_wdt.c b/drivers/watchdog/ipmi_wdt.c
new file mode 100644
index 000000000000..8a60bd78ade4
--- /dev/null
+++ b/drivers/watchdog/ipmi_wdt.c
@@ -0,0 +1,209 @@
+/*
+ * EFI Boot Guard
+ *
+ * Copyright (c) Siemens AG, 2023
+ *
+ * Authors:
+ * Henning Schild <henning...@siemens.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ *
+ * SPDX-License-Identifier: GPL-2.0
+ */
+
+#include <efi.h>
+#include <pci/header.h>
+#include <sys/io.h>
+#include "utils.h"
+
+#define SMBIOS_TYPE_IPMI_KCS 38
+#define IPMI_KCS_DEFAULT_IOBASE 0xca2
+
+#define IPMI_KCS_STS_OBF 0x1
+#define IPMI_KCS_STS_IBF 0x2
+
+#define IPMI_KCS_CMD_ABORT 0x60
+#define IPMI_KCS_CMD_WRITE_START 0x61
+#define IPMI_KCS_CMD_WRITE_END 0x62
+
+#define IPMI_KCS_NETFS_LUN_WDT 0x18
+
+#define IPMI_WDT_CMD_RESET 0x22
+#define IPMI_WDT_CMD_SET 0x24
+#define IPMI_WDT_SET_USE_OSLOAD 0x3
+#define IPMI_WDT_SET_ACTION_HARD_RESET 0x1
+
+#define kcs_sts_is_error(sts) (((sts >> 6 ) & 0x3) == 0x3)
+
+static char
+set_wdt_data[] = {IPMI_WDT_SET_USE_OSLOAD, IPMI_WDT_SET_ACTION_HARD_RESET,
+ 0x00, 0x00,0x00, 0x00};
+
+static EFI_EVENT cmdtimer;
+
+static int probed_before;
+
+static EFI_STATUS
+kcs_wait_iobf(UINT16 io_base, char iobf)
+{
+ EFI_STATUS timerstatus = EFI_NOT_READY;
+ char sts;
+
+ while (timerstatus == EFI_NOT_READY) {
+ sts = inb(io_base + 1);
+ if (kcs_sts_is_error(sts))
+ return EFI_DEVICE_ERROR;
+ if (iobf == IPMI_KCS_STS_IBF) {
+ // IBF we wait for clear
+ if (!(sts & IPMI_KCS_STS_IBF))
+ return EFI_SUCCESS;
+ } else {
+ // OBF we wait for set
+ if (sts & IPMI_KCS_STS_OBF)
+ return EFI_SUCCESS;
+ }
+ uefi_call_wrapper(BS->Stall, 1, (1000 * 1000 / 10));
+ timerstatus = uefi_call_wrapper(BS->CheckEvent, 1, cmdtimer);
+ }
+
+ return EFI_DEVICE_ERROR;
+}
+
+static EFI_STATUS
+kcs_outb(UINT8 value, UINT16 io_base, UINT16 port)
+{
+ EFI_STATUS status;
+
+ status = kcs_wait_iobf(io_base, IPMI_KCS_STS_IBF);
+ if (status)
+ return status;
+
+ outb(value, io_base + port);
+ // dummy read, as mentioned in spec
+ inb(io_base);
+
+ return EFI_SUCCESS;
+}
+
+static EFI_STATUS
+_send_ipmi_cmd(UINT16 io_base, char cmd, char *data, int datalen)
+{
+ int i, err = 0;
+ char lastbyte = cmd;
+
+ err += kcs_outb(IPMI_KCS_CMD_WRITE_START, io_base, 1);
+ err += kcs_outb(IPMI_KCS_NETFS_LUN_WDT, io_base, 0);
+
+ if (datalen) {
+ lastbyte = data[datalen - 1];
+ err += kcs_outb(cmd, io_base, 0);
+ for (i = 0; i < datalen - 1; i++)
+ err += kcs_outb(data[i], io_base, 0);
+ }
+
+ err += kcs_outb(IPMI_KCS_CMD_WRITE_END, io_base, 1);
+ err += kcs_outb(lastbyte, io_base, 0);
+
+ if (err)
+ return EFI_DEVICE_ERROR;
+
+ return kcs_wait_iobf(io_base, IPMI_KCS_STS_OBF);
+}
+
+static VOID
+handle_ipmi_error(UINT16 io_base)
+{
+ WARNING(L"Handling Error Status 0x%x\n", inb(io_base + 1));
+
+ outb(IPMI_KCS_CMD_ABORT, io_base + 1);
+
+ if (kcs_wait_iobf(io_base, IPMI_KCS_STS_IBF))
+ return;
+
+ if (inb((io_base + 1) & IPMI_KCS_STS_OBF))
+ inb(io_base);
+ outb(0x0, io_base);
+
+ if (kcs_wait_iobf(io_base, IPMI_KCS_STS_IBF))
+ return;
+}
+
+static EFI_STATUS
+send_ipmi_cmd(UINT16 io_base, char cmd, char *data, int datalen)
+{
+ EFI_STATUS timerstatus = EFI_NOT_READY;
+ EFI_STATUS status;
+
+ // guard every cmd with a 5s timeout where we retry and try to recover
+ uefi_call_wrapper(BS->SetTimer, 3, cmdtimer, TimerRelative, 50000000);
+
+ while (timerstatus == EFI_NOT_READY) {
+ status = _send_ipmi_cmd(io_base, cmd, data, datalen);
+ if (status == EFI_SUCCESS)
+ return status;
+ handle_ipmi_error(io_base);
+ timerstatus = uefi_call_wrapper(BS->CheckEvent, 1, cmdtimer);
+ }
+
+ return status;
+}
+
+static EFI_STATUS __attribute__((constructor))
+init(__attribute__((unused)) EFI_PCI_IO *pci_io,
+ __attribute__((unused)) UINT16 pci_vendor_id,
+ __attribute__((unused)) UINT16 pci_device_id,
+ UINTN timeout)
+{
+ SMBIOS_STRUCTURE_TABLE *smbios_table;
+ SMBIOS_STRUCTURE_POINTER smbios_struct;
+ EFI_STATUS status;
+ UINT64 io_base;
+ UINT16 *timeout_value;
+
+ // we do not use PCI, and machines with IPMI have many PCI devices
+ if (probed_before++)
+ return EFI_UNSUPPORTED;
+
+ status = LibGetSystemConfigurationTable(&SMBIOSTableGuid,
+ (VOID **)&smbios_table);
+
+ if (status != EFI_SUCCESS)
+ return EFI_UNSUPPORTED;
+
+ smbios_struct = smbios_find_struct(smbios_table, SMBIOS_TYPE_IPMI_KCS);
+
+ if (smbios_struct.Raw == NULL)
+ return EFI_UNSUPPORTED;
+
+ io_base = *((UINT64 *)(smbios_struct.Raw + 8));
+ if (io_base == 0) {
+ io_base = IPMI_KCS_DEFAULT_IOBASE;
+ } else {
+ if (!(io_base & 1))
+ // MMIO not implemented
+ return EFI_UNSUPPORTED;
+
+ io_base &= ~1;
+ }
+
+ INFO(L"Detected IPMI watchdog at I/O 0x%x\n", io_base);
+ timeout_value = (UINT16 *)(set_wdt_data + 4);
+ *timeout_value = timeout * 10;
+
+ status = uefi_call_wrapper(BS->CreateEvent, 5, EVT_TIMER, 0, NULL, NULL, &cmdtimer);
+ if (status != EFI_SUCCESS)
+ return status;
+
+ status = send_ipmi_cmd(io_base, IPMI_WDT_CMD_SET, set_wdt_data,
+ sizeof(set_wdt_data));
+
+ if (status == EFI_SUCCESS)
+ status = send_ipmi_cmd(io_base, IPMI_WDT_CMD_RESET, NULL, 0);
+
+ if (status != EFI_SUCCESS)
+ ERROR(L"Watchdog device repeatedly reported errors.\n");
+
+ uefi_call_wrapper(BS->CloseEvent, 1, cmdtimer);
+ return status;
+}
--
2.39.3

[Jan Kiszka]

Thanks, both applied. This one, I massaged style-wise, please check
again if the result is still fine.

Jan

--
Siemens AG, Technology
Competence Center Embedded Linux

[Jan Kiszka]

From: Henning Schild <henning...@siemens.com>

Tested on a Supermicro X11 series (Simatic IPC 1047E) and on some other
random Supermicro X9 series machine. Should work for any board that has
IPMI in any version. When an IPMI watchdog is present, iTCO is likely
not working. The Linux driver would detect that and has special code to
deal with especially Supermicro, here we use probe order.

Developed using the IPMI Spec 2.0 sections 9 and 27.

Signed-off-by: Henning Schild <henning...@siemens.com>

[Jan: style changes]
Signed-off-by: Jan Kiszka <jan.k...@siemens.com>
---

Had to change more to make cppcheck happy and also switched some var
types. Therefore: v4 (also in next)

Makefile.am | 2 +
drivers/watchdog/ipmi_wdt.c | 213 ++++++++++++++++++++++++++++++++++++
2 files changed, 215 insertions(+)

create mode 100644 drivers/watchdog/ipmi_wdt.c

diff --git a/Makefile.am b/Makefile.am

index 48c560f..ba24400 100644

--- a/Makefile.am
+++ b/Makefile.am
@@ -159,12 +159,14 @@ if BOOTLOADER
if ARCH_IS_X86
# NOTE: wdat.c is placed first so it is tried before any other drivers
# NOTE: ipc4x7e_wdt.c must be *before* itco.c
+# NOTE: ipmi_wdt.c must be *before* itco.c
efi_sources_watchdogs = \
drivers/watchdog/wdat.c \
drivers/watchdog/amdfch_wdt.c \
drivers/watchdog/i6300esb.c \
drivers/watchdog/atom-quark.c \
drivers/watchdog/ipc4x7e_wdt.c \
+ drivers/watchdog/ipmi_wdt.c \
drivers/watchdog/itco.c \
drivers/watchdog/hpwdt.c
else
diff --git a/drivers/watchdog/ipmi_wdt.c b/drivers/watchdog/ipmi_wdt.c
new file mode 100644

index 0000000..f67100d
--- /dev/null
+++ b/drivers/watchdog/ipmi_wdt.c
@@ -0,0 +1,213 @@

+static UINT8

+set_wdt_data[] = {IPMI_WDT_SET_USE_OSLOAD, IPMI_WDT_SET_ACTION_HARD_RESET,
+ 0x00, 0x00,0x00, 0x00};
+
+static EFI_EVENT cmdtimer;
+

+static BOOLEAN probed_before;
+
+static EFI_STATUS
+kcs_wait_iobf(UINT16 io_base, UINTN iobf)

+{
+ EFI_STATUS timerstatus = EFI_NOT_READY;
+

+ while (timerstatus == EFI_NOT_READY) {

+ UINT8 sts = inb(io_base + 1);

+
+ if (kcs_sts_is_error(sts))
+ return EFI_DEVICE_ERROR;
+ if (iobf == IPMI_KCS_STS_IBF) {

+ /* IBF we wait for clear */

+ if (!(sts & IPMI_KCS_STS_IBF))
+ return EFI_SUCCESS;
+ } else {

+ /* OBF we wait for set */

+ if (sts & IPMI_KCS_STS_OBF)
+ return EFI_SUCCESS;
+ }

+ BS->Stall(1000 * 1000 / 10);
+ timerstatus = BS->CheckEvent(cmdtimer);

+ }
+
+ return EFI_DEVICE_ERROR;
+}
+
+static EFI_STATUS
+kcs_outb(UINT8 value, UINT16 io_base, UINT16 port)
+{
+ EFI_STATUS status;
+
+ status = kcs_wait_iobf(io_base, IPMI_KCS_STS_IBF);
+ if (status)
+ return status;
+
+ outb(value, io_base + port);

+ /* dummy read, as mentioned in spec */

+ inb(io_base);
+
+ return EFI_SUCCESS;
+}
+
+static EFI_STATUS

+_send_ipmi_cmd(UINT16 io_base, UINT8 cmd, UINT8 *data, UINTN datalen)
+{
+ UINT8 lastbyte = cmd;
+ UINTN err = 0;

+
+ err += kcs_outb(IPMI_KCS_CMD_WRITE_START, io_base, 1);
+ err += kcs_outb(IPMI_KCS_NETFS_LUN_WDT, io_base, 0);
+
+ if (datalen) {
+ lastbyte = data[datalen - 1];
+ err += kcs_outb(cmd, io_base, 0);

+ for (UINTN n = 0; n < datalen - 1; n++)
+ err += kcs_outb(data[n], io_base, 0);

+send_ipmi_cmd(UINT16 io_base, UINT8 cmd, UINT8 *data, UINTN datalen)

+{
+ EFI_STATUS timerstatus = EFI_NOT_READY;
+ EFI_STATUS status;
+

+ /*
+ * Guard every command with a 5s timeout where we retry and try to
+ * recover.
+ */
+ BS->SetTimer(cmdtimer, TimerRelative, 50000000);

+
+ while (timerstatus == EFI_NOT_READY) {
+ status = _send_ipmi_cmd(io_base, cmd, data, datalen);
+ if (status == EFI_SUCCESS)
+ return status;
+ handle_ipmi_error(io_base);

+ timerstatus = BS->CheckEvent(cmdtimer);

+ }
+
+ return status;
+}
+
+static EFI_STATUS __attribute__((constructor))
+init(__attribute__((unused)) EFI_PCI_IO *pci_io,
+ __attribute__((unused)) UINT16 pci_vendor_id,
+ __attribute__((unused)) UINT16 pci_device_id,
+ UINTN timeout)
+{
+ SMBIOS_STRUCTURE_TABLE *smbios_table;
+ SMBIOS_STRUCTURE_POINTER smbios_struct;
+ EFI_STATUS status;
+ UINT64 io_base;
+ UINT16 *timeout_value;
+

+ /* We do not use PCI, and machines with IPMI have many PCI devices */
+ if (probed_before)
+ return EFI_UNSUPPORTED;
+ probed_before = TRUE;

+
+ status = LibGetSystemConfigurationTable(&SMBIOSTableGuid,
+ (VOID **)&smbios_table);
+
+ if (status != EFI_SUCCESS)
+ return EFI_UNSUPPORTED;
+
+ smbios_struct = smbios_find_struct(smbios_table, SMBIOS_TYPE_IPMI_KCS);
+
+ if (smbios_struct.Raw == NULL)
+ return EFI_UNSUPPORTED;
+
+ io_base = *((UINT64 *)(smbios_struct.Raw + 8));
+ if (io_base == 0) {
+ io_base = IPMI_KCS_DEFAULT_IOBASE;
+ } else {
+ if (!(io_base & 1))

+ /* MMIO not implemented */

+ return EFI_UNSUPPORTED;
+
+ io_base &= ~1;
+ }
+
+ INFO(L"Detected IPMI watchdog at I/O 0x%x\n", io_base);
+ timeout_value = (UINT16 *)(set_wdt_data + 4);
+ *timeout_value = timeout * 10;
+

+ status = BS->CreateEvent(EVT_TIMER, 0, NULL, NULL, &cmdtimer);

+ if (status != EFI_SUCCESS)
+ return status;
+
+ status = send_ipmi_cmd(io_base, IPMI_WDT_CMD_SET, set_wdt_data,
+ sizeof(set_wdt_data));
+
+ if (status == EFI_SUCCESS)
+ status = send_ipmi_cmd(io_base, IPMI_WDT_CMD_RESET, NULL, 0);
+
+ if (status != EFI_SUCCESS)
+ ERROR(L"Watchdog device repeatedly reported errors.\n");
+

+ BS->CloseEvent(cmdtimer);

+ return status;
+}
--

2.35.3

[Henning Schild]

This works just tested it, thanks!

Henning

Am Tue, 6 Jun 2023 22:13:07 +0200
schrieb Jan Kiszka <jan.k...@siemens.com>: