gmirror gm0 destroyed on shutdown; GPT corrupt
Anton Shterenlikht
Following the handbook, section 19.1 RAID1 - mirroring, I'm trying to
use gmirror with 2 identical scsi disks:
da0 at mpt0 bus 0 target 0 lun 0
da0: <SEAGATE ST318452LC 2213> Fixed Direct Access SCSI-3 device
da0: 160.000MB/s transfers (80.000MHz, offset 63, 16bit)
da0: Command Queueing Enabled
da0: 17366MB (35566478 512 byte sectors: 255H 63S/T 2213C)
da1 at mpt0 bus 0 target 1 lun 0
da1: <SEAGATE ST318452LC 2213> Fixed Direct Access SCSI-3 device
da1: 160.000MB/s transfers (80.000MHz, offset 63, 16bit)
da1: Command Queueing Enabled
da1: 17366MB (35566478 512 byte sectors: 255H 63S/T 2213C)
# sysctl kern.geom.debugflags=17
kern.geom.debugflags: 0 -> 17
# gmirror label -vb round-robin gm0 /dev/da0
Metadata value stored on /dev/da0.
Done.
# gmirror load
#g_modevent(MIRROR, LOAD)
g_post_event_x(0xe000000004b8eb10, 0xe000000010686e40, 2, 262144)
g_load_class(MIRROR)
g_mirror_taste(MIRROR, acd0t01)
acd0: FAILURE - READ_BIG MEDIUM ERROR asc=0x02 ascq=0x00
g_detach(0xe0000000106eb580)
g_destroy_consumer(0xe0000000106eb580)
g_destroy_geom(0xe000000010724800(mirror:taste))
g_mirror_taste(MIRROR, acd0)
acd0: FAILURE - READ_BIG MEDIUM ERROR asc=0x02 ascq=0x00
g_detach(0xe0000000106eb700)
g_destroy_consumer(0xe0000000106eb700)
g_destroy_geom(0xe000000010724a00(mirror:taste))
g_mirror_taste(MIRROR, da2p6)
g_detach(0xe0000000106eb880)
g_destroy_consumer(0xe0000000106eb880)
g_destroy_geom(0xe000000010724c00(mirror:taste))
g_mirror_taste(MIRROR, da2p5)
g_detach(0xe0000000106eba80)
g_destroy_consumer(0xe0000000106eba80)
g_destroy_geom(0xe000000010724c00(mirror:taste))
g_mirror_taste(MIRROR, da2p4)
g_detach(0xe0000000106ebc00)
g_destroy_consumer(0xe0000000106ebc00)
g_destroy_geom(0xe000000010630700(mirror:taste))
g_mirror_taste(MIRROR, da2p3)
g_detach(0xe000000010738000)
g_destroy_consumer(0xe000000010738000)
g_destroy_geom(0xe000000010724e00(mirror:taste))
g_mirror_taste(MIRROR, da2p2)
g_detach(0xe000000010738180)
g_destroy_consumer(0xe000000010738180)
g_destroy_geom(0xe00000001075f400(mirror:taste))
g_mirror_taste(MIRROR, da2p1)
g_detach(0xe000000010620e80)
g_destroy_consumer(0xe000000010620e80)
g_destroy_geom(0xe000000010724a00(mirror:taste))
g_mirror_taste(MIRROR, da0p6)
g_detach(0xe0000000106e9580)
g_destroy_consumer(0xe0000000106e9580)
g_destroy_geom(0xe000000010724800(mirror:taste))
g_mirror_taste(MIRROR, da0p5)
g_detach(0xe000000010739700)
g_destroy_consumer(0xe000000010739700)
g_destroy_geom(0xe0000000108f8f00(mirror:taste))
g_mirror_taste(MIRROR, da0p4)
g_detach(0xe000000010739680)
g_destroy_consumer(0xe000000010739680)
g_destroy_geom(0xe0000000108f8d00(mirror:taste))
g_mirror_taste(MIRROR, da0p3)
g_detach(0xe000000010739800)
g_destroy_consumer(0xe000000010739800)
g_destroy_geom(0xe00000001072ce00(mirror:taste))
g_mirror_taste(MIRROR, da0p2)
g_detach(0xe000000010739780)
g_destroy_consumer(0xe000000010739780)
g_destroy_geom(0xe0000000108f8700(mirror:taste))
g_mirror_taste(MIRROR, da0p1)
g_detach(0xe0000000106e9680)
g_destroy_consumer(0xe0000000106e9680)
g_destroy_geom(0xe00000001072cb00(mirror:taste))
g_mirror_taste(MIRROR, da2)
g_detach(0xe0000000106a4900)
g_destroy_consumer(0xe0000000106a4900)
g_destroy_geom(0xe00000001075fa00(mirror:taste))
g_mirror_taste(MIRROR, da1)
g_detach(0xe0000000106ebc80)
g_destroy_consumer(0xe0000000106ebc80)
g_destroy_geom(0xe000000010630b00(mirror:taste))
g_mirror_taste(MIRROR, da0)
g_detach(0xe0000000106e8400)
g_destroy_consumer(0xe0000000106e8400)
g_destroy_geom(0xe00000001072c800(mirror:taste))
g_post_event_x(0xe000000004b861c0, 0xe0000000108f9000, 2, 0)
ref 0xe0000000108f9000
ref 0xe000000010760800
GEOM_MIRROR: Device mirror/gm0 launched (1/1).
g_mirror_taste(MIRROR, ufsid/4a3fa1b76cb317b5)
g_detach(0xe0000000106a4780)
g_destroy_consumer(0xe0000000106a4780)
g_destroy_geom(0xe0000000108fa300(mirror:taste))
g_mirror_taste(MIRROR, ufsid/4a3fa1b69c522d30)
g_detach(0xe0000000106e9900)
g_destroy_consumer(0xe0000000106e9900)
g_destroy_geom(0xe0000000108fac00(mirror:taste))
g_mirror_taste(MIRROR, ufsid/4a3fa1b751514347)
g_detach(0xe0000000106ebb00)
g_destroy_consumer(0xe0000000106ebb00)
g_destroy_geom(0xe00000001075f300(mirror:taste))
g_mirror_taste(MIRROR, ufsid/4a3fa1b5e5003da2)
g_detach(0xe0000000106e9880)
g_destroy_consumer(0xe0000000106e9880)
g_destroy_geom(0xe000000010763600(mirror:taste))
g_mirror_taste(MIRROR, iso9660/FreeBSD_Install)
acd0: FAILURE - READ_BIG MEDIUM ERROR asc=0x02 ascq=0x00
g_detach(0xe000000010738080)
g_destroy_consumer(0xe000000010738080)
g_destroy_geom(0xe00000001081dd00(mirror:taste))
g_mirror_taste(MIRROR, mirror/gm0)
g_detach(0xe0000000106e9980)
g_destroy_consumer(0xe0000000106e9980)
g_destroy_geom(0xe000000010763600(mirror:taste))
dev_taste(DEV,mirror/gm0)
g_part_taste(PART,mirror/gm0)
GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
GEOM: mirror/gm0: using the primary only -- recovery suggested.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
g_post_event_x(0xe000000004b861c0, 0xe0000000108fa400, 2, 0)
ref 0xe0000000108fa400
ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe00000001075fa00, 2, 0)
ref 0xe00000001075fa00
ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe0000000108f8700, 2, 0)
ref 0xe0000000108f8700
ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe0000000108f8500, 2, 0)
ref 0xe0000000108f8500
ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe000000010725a00, 2, 0)
ref 0xe0000000108f8500
ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe000000010725a00, 2, 0)
ref 0xe000000010725a00
ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe000000010631500, 2, 0)
ref 0xe000000010631500
ref 0xe0000000108f8300
g_label_taste(LABEL, mirror/gm0)
g_detach(0xe000000010738080)
g_destroy_consumer(0xe000000010738080)
g_destroy_geom(0xe00000001075ef00(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p1)
g_detach(0xe000000010739b00)
g_destroy_consumer(0xe000000010739b00)
g_destroy_geom(0xe000000010762f00(mirror:taste))
dev_taste(DEV,mirror/gm0p1)
g_part_taste(PART,mirror/gm0p1)
g_wither_geom(0xe000000010763a00(mirror/gm0p1))
g_label_taste(LABEL, mirror/gm0p1)
g_detach(0xe0000000108fc000)
g_destroy_consumer(0xe0000000108fc000)
g_destroy_geom(0xe00000001081d500(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p2)
g_detach(0xe0000000108fc100)
g_destroy_consumer(0xe0000000108fc100)
g_destroy_geom(0xe0000000108f9d00(mirror:taste))
dev_taste(DEV,mirror/gm0p2)
g_part_taste(PART,mirror/gm0p2)
g_wither_geom(0xe00000001072c800(mirror/gm0p2))
g_label_taste(LABEL, mirror/gm0p2)
g_slice_config(mirror/gm0p2, 0, 1)
g_post_event_x(0xe000000004b861c0, 0xe0000000108fae00, 2, 0)
ref 0xe0000000108fae00
ref 0xe0000000108faa00
g_detach(0xe0000000108fc300)
g_destroy_consumer(0xe0000000108fc300)
g_destroy_geom(0xe00000001081d100(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p3)
g_detach(0xe0000000108fc380)
g_destroy_consumer(0xe0000000108fc380)
g_destroy_geom(0xe000000010724600(mirror:taste))
dev_taste(DEV,mirror/gm0p3)
g_part_taste(PART,mirror/gm0p3)
g_wither_geom(0xe00000001075ed00(mirror/gm0p3))
g_label_taste(LABEL, mirror/gm0p3)
g_detach(0xe0000000106e9900)
g_destroy_consumer(0xe0000000106e9900)
g_destroy_geom(0xe000000010630700(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p4)
g_detach(0xe0000000108fc600)
g_destroy_consumer(0xe0000000108fc600)
g_destroy_geom(0xe0000000108fad00(mirror:taste))
dev_taste(DEV,mirror/gm0p4)
g_part_taste(PART,mirror/gm0p4)
g_wither_geom(0xe000000010762a00(mirror/gm0p4))
g_label_taste(LABEL, mirror/gm0p4)
g_slice_config(mirror/gm0p4, 0, 1)
g_post_event_x(0xe000000004b861c0, 0xe000000010630c00, 2, 0)
ref 0xe000000010630c00
ref 0xe0000000108f8c00
g_detach(0xe0000000108fc800)
g_destroy_consumer(0xe0000000108fc800)
g_destroy_geom(0xe000000010763900(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p5)
g_detach(0xe0000000108fcb00)
g_destroy_consumer(0xe0000000108fcb00)
g_destroy_geom(0xe000000010724800(mirror:taste))
dev_taste(DEV,mirror/gm0p5)
g_part_taste(PART,mirror/gm0p5)
g_wither_geom(0xe000000010724e00(mirror/gm0p5))
g_label_taste(LABEL, mirror/gm0p5)
g_slice_config(mirror/gm0p5, 0, 1)
g_post_event_x(0xe000000004b861c0, 0xe0000000108f8e00, 2, 0)
ref 0xe0000000108f8e00
ref 0xe00000001075f400
g_detach(0xe000000010739780)
g_destroy_consumer(0xe000000010739780)
g_destroy_geom(0xe00000001072c400(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p6)
g_detach(0xe0000000108fcb80)
g_destroy_consumer(0xe0000000108fcb80)
g_destroy_geom(0xe00000001081d500(mirror:taste))
dev_taste(DEV,mirror/gm0p6)
g_part_taste(PART,mirror/gm0p6)
g_wither_geom(0xe0000000108f8200(mirror/gm0p6))
g_label_taste(LABEL, mirror/gm0p6)
g_slice_config(mirror/gm0p6, 0, 1)
g_post_event_x(0xe000000004b861c0, 0xe00000001081d100, 2, 0)
ref 0xe00000001081d100
ref 0xe0000000108f9d00
g_detach(0xe0000000108fcc80)
g_destroy_consumer(0xe0000000108fcc80)
g_destroy_geom(0xe0000000108fb300(label:taste))
g_mirror_taste(MIRROR, ufsid/4a43468a267a63a6)
g_detach(0xe0000000108fcd80)
g_destroy_consumer(0xe0000000108fcd80)
g_destroy_geom(0xe000000010724600(mirror:taste))
dev_taste(DEV,ufsid/4a43468a267a63a6)
g_part_taste(PART,ufsid/4a43468a267a63a6)
g_wither_geom(0xe000000010724600(ufsid/4a43468a267a63a6))
g_label_taste(LABEL, ufsid/4a43468a267a63a6)
g_mirror_taste(MIRROR, ufsid/4a43468cf6208bf0)
g_detach(0xe000000010739480)
g_destroy_consumer(0xe000000010739480)
g_destroy_geom(0xe00000001081d500(mirror:taste))
dev_taste(DEV,ufsid/4a43468cf6208bf0)
g_part_taste(PART,ufsid/4a43468cf6208bf0)
g_wither_geom(0xe000000010724800(ufsid/4a43468cf6208bf0))
g_label_taste(LABEL, ufsid/4a43468cf6208bf0)
g_mirror_taste(MIRROR, ufsid/4a43468b228d030c)
g_detach(0xe0000000106eaf80)
g_destroy_consumer(0xe0000000106eaf80)
g_destroy_geom(0xe00000001081c200(mirror:taste))
dev_taste(DEV,ufsid/4a43468b228d030c)
g_part_taste(PART,ufsid/4a43468b228d030c)
g_wither_geom(0xe00000001081dc00(ufsid/4a43468b228d030c))
g_label_taste(LABEL, ufsid/4a43468b228d030c)
g_mirror_taste(MIRROR, ufsid/4a43468c8715f453)
g_detach(0xe0000000108fd100)
g_destroy_consumer(0xe0000000108fd100)
g_destroy_geom(0xe0000000108f8100(mirror:taste))
dev_taste(DEV,ufsid/4a43468c8715f453)
g_part_taste(PART,ufsid/4a43468c8715f453)
g_wither_geom(0xe000000010724a00(ufsid/4a43468c8715f453))
g_label_taste(LABEL, ufsid/4a43468c8715f453)
g_detach(0xe0000000106ea680)
g_destroy_consumer(0xe0000000106ea680)
g_destroy_geom(0xe000000010724a00(ufsid/4a43468c8715f453))
g_detach(0xe0000000108fd000)
g_destroy_consumer(0xe0000000108fd000)
g_destroy_geom(0xe00000001081dc00(ufsid/4a43468b228d030c))
g_detach(0xe000000010739280)
g_destroy_consumer(0xe000000010739280)
g_destroy_geom(0xe000000010724800(ufsid/4a43468cf6208bf0))
g_detach(0xe0000000106eb880)
g_destroy_consumer(0xe0000000106eb880)
g_destroy_geom(0xe000000010724600(ufsid/4a43468a267a63a6))
g_detach(0xe000000010738000)
g_destroy_consumer(0xe000000010738000)
g_destroy_geom(0xe0000000108f8200(mirror/gm0p6))
g_detach(0xe0000000106a4900)
g_destroy_consumer(0xe0000000106a4900)
g_destroy_geom(0xe000000010724e00(mirror/gm0p5))
g_detach(0xe0000000108fc700)
g_destroy_consumer(0xe0000000108fc700)
g_destroy_geom(0xe000000010762a00(mirror/gm0p4))
g_detach(0xe0000000108fc480)
g_destroy_consumer(0xe0000000108fc480)
g_destroy_geom(0xe00000001075ed00(mirror/gm0p3))
g_detach(0xe0000000108fc200)
g_destroy_consumer(0xe0000000108fc200)
g_destroy_geom(0xe00000001072c800(mirror/gm0p2))
g_detach(0xe000000010739c80)
g_destroy_consumer(0xe000000010739c80)
g_destroy_geom(0xe000000010763a00(mirror/gm0p1))
# ls -al /dev/mirror/
total 1
dr-xr-xr-x 2 root wheel 512 Jun 25 11:16 .
dr-xr-xr-x 9 root wheel 512 Jun 25 11:22 ..
crw-r----- 1 root operator 0, 105 Jun 25 11:16 gm0
crw-r----- 1 root operator 0, 116 Jun 25 11:16 gm0p1
crw-r----- 1 root operator 0, 117 Jun 25 11:16 gm0p2
crw-r----- 1 root operator 0, 118 Jun 25 11:16 gm0p3
crw-r----- 1 root operator 0, 119 Jun 25 11:16 gm0p4
crw-r----- 1 root operator 0, 120 Jun 25 11:16 gm0p5
crw-r----- 1 root operator 0, 121 Jun 25 11:16 gm0p6
#
#echo 'geom_mirror_load="YES"' >> /boot/loader.conf
# cat /boot/loader.conf
vfs.root.mountfrom="ufs:/dev/da0p2"
geom_mirror_load="YES"
#
On shutdown I see on console:
[skip]
g_detach(0xe0000000106c7000)
g_destroy_consumer(0xe0000000106c7000)
g_destroy_geom(0xe0000000106c3200(mirror/gm0p2))
g_destroy_geom(0xe000000010828200(mirror/gm0))
g_post_event_x(0xe000000004c33e70, 0xe0000000106c6d80, 2, 0)
g_wither_geom(0xe0000000106efa00(gm0.sync))
GEOM_MIRROR: Device gm0 destroyed.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
g_wither_geom(0xe000000010826900(gm0))
g_detach(0xe0000000106c6d80)
g_destroy_consumer(0xe0000000106c6d80)
g_destroy_geom(0xe0000000106efa00(gm0.sync))
g_destroy_geom(0xe000000010826900(gm0))
And when the system is rebooted, there is no /dev/mirror anymore.
Please advise
many thanks
--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: +44 (0)117 928 8233
Fax: +44 (0)117 929 4423
_______________________________________________
freebsd-...@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questi...@freebsd.org"
Marcel Moolenaar
On Jun 25, 2009, at 4:02 AM, Anton Shterenlikht wrote:
> dev_taste(DEV,mirror/gm0)
> g_part_taste(PART,mirror/gm0)
>
> GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
> GEOM: mirror/gm0: using the primary only -- recovery suggested.
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
You created the mirror after the GPT, which means you destroyed
the GPT backup header. gmirror uses the last sector on the disk
for metadata and that by itself is a cause for various problems.
It's better to use gmirror per partition.
>
> #echo 'geom_mirror_load="YES"' >> /boot/loader.conf
Is /boot a symlink for /efi/boot?
> GEOM_MIRROR: Device gm0 destroyed.
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
This is normal.
>
> And when the system is rebooted, there is no /dev/mirror anymore.
You could run into a race condition between GPT and gmirror and
GPT winning (again the result of gmirror using the last sector
on a disk for metadata).
Alternatively, make sure gmirror got loaded at boot.
FYI,
--
Marcel Moolenaar
xcl...@mac.com
Marcel Moolenaar
On Jun 27, 2009, at 4:15 AM, Ivan Voras wrote:
> Marcel Moolenaar wrote:
>> On Jun 25, 2009, at 4:02 AM, Anton Shterenlikht wrote:
>>> dev_taste(DEV,mirror/gm0)
>>> g_part_taste(PART,mirror/gm0)
>>>
>>> GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
>>> GEOM: mirror/gm0: using the primary only -- recovery suggested.
>>> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>> You created the mirror after the GPT, which means you destroyed
>> the GPT backup header. gmirror uses the last sector on the disk
>> for metadata and that by itself is a cause for various problems.
>> It's better to use gmirror per partition.
>
> Or create the GPT partition inside the gmirror device - then the GPT
> backup table will be at last_sector-1, but...
>
>> You could run into a race condition between GPT and gmirror and
>> GPT winning (again the result of gmirror using the last sector
>> on a disk for metadata).
>
> unfortunately this could still happen, and will lead to the same
> error if GPT is tasted first, since it is embedded in the first
> sector and will assume the whole drive is available to GPT, and will
> then proceed to not find its backup data in the last sector.
>
> It looks to me like GEOM classes should have a "priority" field for
> tasting. Any objections to that idea?
Using the last sector is not only flawed because it creates a race
condition, it's flawed in the assumption that you can always make
a geom part of a mirror by storing meta-data on the geom without
causing corruption. This whole idea of using the last sector was
so that a fully partitioned disk with data could be turned into a
mirrored disk. A neat idea, but hardly the basis for a generic
mirroring implementation when it silently corrupts a disk.
I think it's better to change gmirror to use the first sector on the
provider. This never creates a race condition and as such, you don't
need to invent a priority scheme, that has it's own set of flaws on
top of it. The only downside is that it's not easy to make a fully
partitioned and populated disk part of a mirror: one would need to
move the data forward one sector to free the first sector. This we
can actually do by inserting a GEOM that does it while I/O is still
ongoing. The good thing is: we need a class that does exactly this
for implementing the "move" verb in gpart.
In other words: Solving the problem that putting the metadata in the
first sector creates, can and will be re-used in implementing the
gpart "move partition" feature. I doubt anyone will complain that
the creation of a mirror brings with it a few hours of disk activity
that does not inhibit normal operation...
Pawel Jakub Dawidek
> Using the last sector is not only flawed because it creates a race
> condition, it's flawed in the assumption that you can always make
> a geom part of a mirror by storing meta-data on the geom without
> causing corruption. This whole idea of using the last sector was
> so that a fully partitioned disk with data could be turned into a
> mirrored disk. A neat idea, but hardly the basis for a generic
> mirroring implementation when it silently corrupts a disk.
This wasn't the idea:) People started putting gmirror on top of
partitioned disk, because it was easier/simpler/faster than creating
mirror, partitioning and copying the data. I for one never put mirror on
already partitioned disk. Although it is sometimes safe to use the last
sector. Gjournal already looks for UFS and if UFS is in place, it
figures out if the last sector is in use - it isn't if partition size is
not multiple of UFS block size.
> I think it's better to change gmirror to use the first sector on the
> provider. This never creates a race condition and as such, you don't
> need to invent a priority scheme, that has it's own set of flaws on
> top of it. The only downside is that it's not easy to make a fully
> partitioned and populated disk part of a mirror: one would need to
> move the data forward one sector to free the first sector. This we
> can actually do by inserting a GEOM that does it while I/O is still
> ongoing. The good thing is: we need a class that does exactly this
> for implementing the "move" verb in gpart.
There were two reasons to use the last sector instead of first:
1. You want to be able to boot from gmirror. If all your data will be
moved forward your boot sectors and kernel will be harder to find.
2. For recovery reasons you may want to turn off gmirror and still be
able to access your data.
Note that gmirror can handle the case where disk, slice and partition
share the same last sector - it simply stores provider size in its
metadata, so once it gets disk for tasting it detects its too big and
ignores it, then slice will be given for tasting, but it also has larger
size than expected and will be ignored as well. Finally partition will
be tasted and gmirror configured.
--
Pawel Jakub Dawidek http://www.wheel.pl
p...@FreeBSD.org http://www.FreeBSD.org
FreeBSD committer Am I Evil? Yes, I Am!
Ivan Voras
> Using the last sector is not only flawed because it creates a race
> condition, it's flawed in the assumption that you can always make
> a geom part of a mirror by storing meta-data on the geom without
> causing corruption. This whole idea of using the last sector was
> so that a fully partitioned disk with data could be turned into a
> mirrored disk. A neat idea, but hardly the basis for a generic
> mirroring implementation when it silently corrupts a disk.
>
> I think it's better to change gmirror to use the first sector on the
> provider.
Yes, it would be cleaner to implement but it would also make the
mirrored devices unbootable.
But maybe the class of users needing the functionality is smaller now.
> This never creates a race condition and as such, you don't
> need to invent a priority scheme, that has it's own set of flaws on
> top of it. The only downside is that it's not easy to make a fully
> partitioned and populated disk part of a mirror: one would need to
> move the data forward one sector to free the first sector. This we
> can actually do by inserting a GEOM that does it while I/O is still
> ongoing. The good thing is: we need a class that does exactly this
> for implementing the "move" verb in gpart.
Looks too complicated and fragile. Maybe there's a need for
metadata-less automatic mirrors in some way, by storing the
configuration somewhere else, possibly in /etc.
Aisaka Taiga
> Yes, it would be cleaner to implement but it would also make the
> mirrored devices unbootable.
> But maybe the class of users needing the functionality is smaller now.
>
systems because that would drastically increase the price of a single
system; yet many customers want mirroring.
> Looks too complicated and fragile. Maybe there's a need for
> metadata-less automatic mirrors in some way, by storing the
> configuration somewhere else, possibly in /etc.
swapped; such a configuration might lead to data corruption on a volume
which was enumerated incorrectly or swapped.
--
Kamigishi Rei
KREI-RIPE
Anton Shterenlikht
>
> On Jun 25, 2009, at 4:02 AM, Anton Shterenlikht wrote:
> > dev_taste(DEV,mirror/gm0)
> > g_part_taste(PART,mirror/gm0)
> >
> > GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
> > GEOM: mirror/gm0: using the primary only -- recovery suggested.
> > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>
> You created the mirror after the GPT, which means you destroyed
> the GPT backup header. gmirror uses the last sector on the disk
> for metadata and that by itself is a cause for various problems.
So, gmirror cannot be used on ia64 to mirror the boot disk?
Because on ia64 the last sector always contains secondary GPT.
I take it the RAID1 section, 19.4, in FBSD user manual,
was written with i386 or alpha architecture in mind.
> It's better to use gmirror per partition.
how? Is it in the manual? any link?
> > #echo 'geom_mirror_load="YES"' >> /boot/loader.conf
>
> Is /boot a symlink for /efi/boot?
yes,
lrwxr-xr-x 1 root wheel 8 Jun 25 10:44 boot -> efi/boot
> > And when the system is rebooted, there is no /dev/mirror anymore.
>
> You could run into a race condition between GPT and gmirror and
> GPT winning (again the result of gmirror using the last sector
> on a disk for metadata).
>
> Alternatively, make sure gmirror got loaded at boot.
# kldstat
Id Refs Address Size Name
1 3 0xe000000004000000 ff9c08 kernel
2 1 0xe000000004ffa000 3c830 geom_mirror.ko
#
It's not that I desperately need to mirror a boot disk, it just that
gmirror looked so easy in the manual, I wanted to give it a go.
Perhaps I can just do a block copy to the second disk, say once a day,
and have it as a backup.
Could you also possibly comment on gvinum on ia64?
many thanks
anton
--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: +44 (0)117 928 8233
Fax: +44 (0)117 929 4423
Lorenzo Perone
> On 28.06.2009, at 10:49, Pawel Jakub Dawidek wrote:
>
>> .... I for one never put mirror on
>> already partitioned disk. Although it is sometimes safe to use the
>> last
>> sector. Gjournal already looks for UFS and if UFS is in place, it
>> figures out if the last sector is in use - it isn't if partition
>> size is
>> not multiple of UFS block size.
>>
does this actually also mean that gmirror used on a partition
(eg mirroring two partitions of two different disks) is
not recommended and is going to write its metadata always
on the last sector of the disk, instead of the last sector of
the partition?
regards,
Lorenzo
Anton Shterenlikht
>
> > dev_taste(DEV,mirror/gm0)
> > g_part_taste(PART,mirror/gm0)
> >
> > GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
> > GEOM: mirror/gm0: using the primary only -- recovery suggested.
> > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>
> You created the mirror after the GPT, which means you destroyed
> the GPT backup header. gmirror uses the last sector on the disk
> for metadata and that by itself is a cause for various problems.
>
> It's better to use gmirror per partition.
Like this?
# gmirror label -vb round-robin root /dev/da0p2
gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
#
I've read some boot disk gmirror examples, e.g.
http://people.freebsd.org/~rse/mirror
however, all examples I've seen are for i386, talking about MBR, fdisk
and bsdlabel, so these are not directly applicable to ia64.
Application of gvinum for boot disk on ia64 is not clear either.
It seems gvinum section of the handbook, 21.9, is also based on i386.
Please advise
many thanks
anton
--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: +44 (0)117 928 8233
Fax: +44 (0)117 929 4423
Wojciech Puchar
>
> Like this?
>
> # gmirror label -vb round-robin root /dev/da0p2
> gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
Anton Shterenlikht
> >> It's better to use gmirror per partition.
> >
> > Like this?
> >
> > # gmirror label -vb round-robin root /dev/da0p2
> > gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
> isn't that partition accessed by other process or mounted?
should it not be mounted?
Sorry, I was just following the handbook, but I now understand it is
incorrect when it comes to ia64.
many thanks
anton
> _______________________________________________
> freebsd-...@freebsd.org mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-questions
> To unsubscribe, send any mail to "freebsd-questi...@freebsd.org"
--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: +44 (0)117 928 8233
Fax: +44 (0)117 929 4423
Wojciech Puchar
>>> # gmirror label -vb round-robin root /dev/da0p2
>>> gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
>> isn't that partition accessed by other process or mounted?
>
> should it not be mounted?
Anton Shterenlikht
> >>>
> >>> # gmirror label -vb round-robin root /dev/da0p2
> >>> gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
> >> isn't that partition accessed by other process or mounted?
> >
> > should it not be mounted?
> yes it should not, no matter what architecture.
ok, thank you
So how can I gmirror root partition? I can't unmount it, I think.
Perhaps I need to use a single-user mode?
Following is a gpart/gmirror report - some success and problems.
I did a fresh FBSD current install on ia64 on directly attached scsi, da0.
# gpart show
=> 34 35566411 da0 GPT (17G)
34 819200 1 efi (400M)
819234 1048576 2 freebsd-ufs (512M)
1867810 4194304 3 freebsd-swap (2.0G)
6062114 2097152 4 freebsd-ufs (1.0G)
8159266 2097152 5 freebsd-ufs (1.0G)
10256418 25310027 6 freebsd-ufs (12G)
#
What I want is to mirror the whole of the boot disk to da1, which
is identical to da0, but following Marcel's advice, will apply
gmirror per partition.
So starting with efi partition:
First I create GPT scheme on da1
# gpart create -s gpt da1
da1 created
# gpart show da1
=> 34 35566411 da1 GPT (17G)
34 35566411 - free - (17G)
#
then I create EFI partition of the same size as on the boot disk, da0.
# gpart add -b 34 -s 819200 -t efi da1
da1p1 added
# gpart show da1
=> 34 35566411 da1 GPT (17G)
34 819200 1 efi (400M)
819234 34747211 - free - (17G)
#
then I umount /efi so that I can create gmirror label on da0p1.
# umount /efi
# gmirror label -vb round-robin efi /dev/da0p1
Metadata value stored on /dev/da0p1.
Done.
#
Checking gmirror
# gmirror status
Name Status Components
mirror/efi COMPLETE da0p1
#
and another check
# gmirror list
Geom name: efi
State: COMPLETE
Components: 1
Balance: round-robin
Slice: 4096
Flags: NONE
GenID: 0
SyncID: 1
ID: 3904698645
Providers:
1. Name: mirror/efi
Mediasize: 419429888 (400M)
Sectorsize: 512
Mode: r0w0e0
Consumers:
1. Name: da0p1
Mediasize: 419430400 (400M)
Sectorsize: 512
Mode: r1w1e1
State: ACTIVE
Priority: 0
Flags: NONE
GenID: 0
SyncID: 1
ID: 1288665799
#
now insert a spare partition, da1p1, into the mirror
# gmirror insert efi /dev/da1p1
status looks fine
# gmirror status
Name Status Components
mirror/efi DEGRADED da0p1
da1p1 (44%)
# gmirror status
Name Status Components
mirror/efi DEGRADED da0p1
da1p1 (87%)
# gmirror status
Name Status Components
mirror/efi COMPLETE da0p1
da1p1
#
and another check
# gmirror list
Geom name: efi
State: COMPLETE
Components: 2
Balance: round-robin
Slice: 4096
Flags: NONE
GenID: 0
SyncID: 1
ID: 3904698645
Providers:
1. Name: mirror/efi
Mediasize: 419429888 (400M)
Sectorsize: 512
Mode: r0w0e0
Consumers:
1. Name: da0p1
Mediasize: 419430400 (400M)
Sectorsize: 512
Mode: r1w1e1
State: ACTIVE
Priority: 0
Flags: NONE
GenID: 0
SyncID: 1
ID: 1288665799
2. Name: da1p1
Mediasize: 419430400 (400M)
Sectorsize: 512
Mode: r1w1e1
State: ACTIVE
Priority: 0
Flags: NONE
GenID: 0
SyncID: 1
ID: 1724596009
#
So far, so good.
Now, I don't need to create the filesystem on the mirror, because EFI
was copied from da0p1 to da1p1.
So, I try to mount /dev/mirror/efi
# mount -t msdosfs /dev/mirror/efi /mnt
# df
Filesystem 1K-blocks Used Avail Capacity Mounted on
/dev/da0p2 507630 35904 431116 8% /
devfs 1 1 0 100% /dev
/dev/da0p5 1012974 12 931926 0% /tmp
/dev/da0p6 12252370 252608 11019574 2% /usr
/dev/da0p4 1012974 242 931696 0% /var
/dev/mirror/efi 409504 163264 246240 40% /mnt
#
again seems ok
so I proceed to modify /etc/fstab and change da0p1 into mirror/efi
# cat /etc/fstab
# Device Mountpoint FStype Options Dump Pass#
/dev/da0p3 none swap sw 0 0
/dev/da0p2 / ufs rw 1 1
/dev/mirror/efi /efi msdosfs rw 0 0
^^^^^^^^^^^^^^^
/dev/da0p5 /tmp ufs rw 2 2
/dev/da0p6 /usr ufs rw 2 2
/dev/da0p4 /var ufs rw 2 2
/dev/acd0 /cdrom cd9660 ro,noauto 0 0
#
now I can try to just mount /efi
# umount /mnt
# mount /efi
# df
Filesystem 1K-blocks Used Avail Capacity Mounted on
/dev/da0p2 507630 35904 431116 8% /
devfs 1 1 0 100% /dev
/dev/da0p5 1012974 12 931926 0% /tmp
/dev/da0p6 12252370 252608 11019574 2% /usr
/dev/da0p4 1012974 242 931696 0% /var
/dev/mirror/efi 409504 163264 246240 40% /efi
#
good, working!
now to mirror root partition.
My problem is that root is mounted and cannot (?) be unmounted, unlike /efi,
on the live system.
# gpart add -b 819234 -s 1048576 -t freebsd-ufs da1
da1p2 added
#
# gmirror label -vb round-robin root /dev/da0p2
gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
#
If I create gmirror on da1, the spare disk:
# gmirror label -vb round-robin root /dev/da1p2
Metadata value stored on /dev/da0p1.
Done.
#
so that
# gmirror status
Name Status Components
mirror/efi COMPLETE da0p1
da1p1
mirror/root COMPLETE da1p2
#
then I still cannot insert da0p2
# gmirror insert root da0p2
gmirror: Cannot access provider da0p2.
#
So how can I gmirror root partion on a live system?
Also, if gmirror overwrites the last sector in the partition,
then gmirror per partition cannot be used for /usr, because
secondary GPT is stored in the last sector of /usr. Isn't it?
many thanks
--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: +44 (0)117 928 8233
Fax: +44 (0)117 929 4423
Alban Hertroys
You're almost there... I did this a while ago, can't remember when,
but I just upgraded the system that had this from FreeBSD 6.3 of
sometime in 2006 to 7.2.
What I believe I did from this point on was:
Copy everything from the root partition to mirror/root.
Modify /etc/fstab to mount root on mirror/root.
Reboot.
Now the original root partition isn't mounted anymore, so we can do
operate on it's geom stuff.
gmirror insert root da0p2
That should be it.
If that doesn't work you can always boot off a live file-system CD/DVD
and perform these actions from there. You won't have man pages in that
case though, or at least I couldn't find a way to read them off the
DVD last I tried.
One thing I'd like to warn about at this point:
If you ever upgrade to a kernel with a newer geom metadata version and
that new kernel crashes, you're left with a system where the new
kernel can't boot at all while the old kernel can't mount the root
mirror as it's now of a version it can't handle.
You can however mount a single geom provider of that root file system
(/dev/da1p2 for example) to try to fix things.
That file-system WILL be dirty, but DON'T run fsck on it or you will
destroy it's contents. That's what happened to my upgrade above...
Thankfully it was only my root partition with hardly any data on it
and I did make level 0 dumps before the upgrade, but I needed to
restore that FS from a fixit shell without man pages. Augh!
> many thanks
>
> --
> Anton Shterenlikht
> Room 2.6, Queen's Building
> Mech Eng Dept
> Bristol University
> University Walk, Bristol BS8 1TR, UK
> Tel: +44 (0)117 928 8233
> Fax: +44 (0)117 929 4423
> _______________________________________________
> freebsd...@freebsd.org mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-current
> To unsubscribe, send any mail to "freebsd-curre...@freebsd.org
> "
>
>
>
>
Alban Hertroys
--
If you can't see the forest for the trees,
cut the trees and you'll see there is no forest.
!DSPAM:760,4a4de90f759155226611503!
Anton Shterenlikht
thank you, that was helpful.
I think I've got it, but it's a bit more complex on ia64
because /boot is a symlink to /efi/boot, which is a separate
partition.
Anyway, I've got:
# gmirror status
Name Status Components
mirror/efi COMPLETE da0p1
da1p1
mirror/root COMPLETE da0p2
da1p2
mirror/swap COMPLETE da0p3
da1p3
mirror/var COMPLETE da1p4
da0p4
mirror/tmp COMPLETE da1p5
da0p5
mirror/usr DEGRADED da1p6
da0p6 (24%)
#
I'll try to write up my experience and post later.
thanks again
--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: +44 (0)117 928 8233
Fax: +44 (0)117 929 4423
_______________________________________________