Thanks for the hints and for the link to documentation. In the meantime I've figured it out by looking at how other plugins do that. The trick is that it's essentially a two-step process, first I needed to add a menu to the itemmenu, then add a popup to this menu. There I could finally add the individual sub items/commands.
// let's add a submenu
let my_submenu = doc.createElementNS(XUL_NS, 'menu');
my_submenu.id = 'my_submenu';
my_submenu.setAttribute('type', 'menu');
my_submenu.setAttribute('class', 'menuitem-iconic');
my_submenu.setAttribute('data-l10n-id', 'my_submenu');
doc.getElementById('zotero-itemmenu').appendChild(my_submenu);
this.storeAddedElement(my_submenu);
// let's a popup to the submenu
let my_popup = my_submenu.appendChild(doc.createElementNS(XUL_NS, 'menupopup'));
my_popup.id = 'my_popup';
this.storeAddedElement(my_popup);
// finally we can add our menu items to the popup
let submenuitem = doc.createElementNS(XUL_NS, 'menuitem');
submenuitem.id = 'submenuitem';
submenuitem.setAttribute('type', 'command');
submenuitem.setAttribute('class', 'menuitem');
submenuitem.setAttribute('data-l10n-id', 'my-submenu-item');
submenuitem.addEventListener('command', () => {
// do something
this.whatever();
});
my_popup.appendChild(submenuitem);
this.storeAddedElement(submenuitem);
// add another item in the popup
let submenuitem2 = doc.createElementNS(XUL_NS, 'menuitem');
submenuitem2.id = 'submenuitem2';
submenuitem2.setAttribute('type', 'command');
submenuitem2.setAttribute('class', 'menuitem');
submenuitem2.setAttribute('data-l10n-id', 'my-submenu-item2');
submenuitem2.addEventListener('command', () => {
// do something
this.whatever();
});
my_popup.appendChild(submenuitem2);
this.storeAddedElement(submenuitem2);