[PATCH 2/2] md: add fsid check for sheep store disks to handle mountpoint loss

[guihe...@cmss.chinamobile.com]

From: Gui Hecheng <guihe...@cmss.chinamobile.com>

As we encountered, systemd may umount a disk when it detects certain errors,
then sheep fall back to use the disk(often the system disk) of the mountpoint
directory as a store disk.
As follows:
before
Id Size Used Avail Use% Path
0 3.6 GB 1.0 GB 2.5 GB 28% /mnt/sheep1
1 3.6 GB 296 MB 3.3 GB 8% /mnt/sheep2 <-- original disk
after
Id Size Used Avail Use% Path
0 3.6 GB 1.0 GB 2.5 GB 28% /mnt/sheep1
1 12 GB 0.0 MB 12 GB 0% /mnt/sheep2 <-- fall to system disk

Upon this case, sheep fails to tell that the original disk is gone just
based on the directory name.

We add a fsid field to struct disk to check for mountpoint loss.
After the change, the disk sheep fall backed on will be correctly unpluged.

Signed-off-by: Gui Hecheng <guihe...@cmss.chinamobile.com>
---
sheep/sheep_priv.h | 2 ++
sheep/store/common.c | 8 ++++++--
sheep/store/md.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/sheep/sheep_priv.h b/sheep/sheep_priv.h
index b4e8ad9..555e868 100644
--- a/sheep/sheep_priv.h
+++ b/sheep/sheep_priv.h
@@ -173,6 +173,7 @@ struct disk {
struct rb_node rb;
char path[PATH_MAX];
uint64_t space;
+ uint64_t fsid;
};

struct vdisk {
@@ -546,6 +547,7 @@ int md_plug_disks(char *disks);
int md_unplug_disks(char *disks);
uint64_t md_get_size(uint64_t *used);
uint32_t md_nr_disks(void);
+bool md_verify_disk(const char *path);

static inline bool is_stale_path(const char *path)
{
diff --git a/sheep/store/common.c b/sheep/store/common.c
index ecd3c84..c55eec3 100644
--- a/sheep/store/common.c
+++ b/sheep/store/common.c
@@ -52,9 +52,13 @@ static char *get_obj_dir(char *path)
return dirname(path);
}

+static int is_dir_corrupted(const char *dir)
+{
+ return !md_verify_disk(dir);
+}
+
int err_to_sderr(const char *path, uint64_t oid, int err)
{
- struct stat s;
char p[PATH_MAX], *dir;

/* Use a temporary buffer since dirname() may modify its argument. */
@@ -64,7 +68,7 @@ int err_to_sderr(const char *path, uint64_t oid, int err)
sd_debug("%s", path);
switch (err) {
case ENOENT:
- if (stat(dir, &s) < 0) {
+ if (is_dir_corrupted(dir)) {
sd_err("%s corrupted", dir);
return md_handle_eio(dir);
}
diff --git a/sheep/store/md.c b/sheep/store/md.c
index dd3d8d4..616dd3d 100644
--- a/sheep/store/md.c
+++ b/sheep/store/md.c
@@ -321,6 +321,17 @@ broken_path:
return 0;
}

+static uint64_t init_path_fsid(const char *path)
+{
+ struct statvfs fs;
+
+ if (statvfs(path, &fs) < 0) {
+ sd_err("get disk %s fsid failed %m", path);
+ return 0;
+ }
+ return fs.f_fsid;
+}
+
/* We don't need lock at init stage */
bool md_add_disk(const char *path, bool purge)
{
@@ -345,6 +356,12 @@ bool md_add_disk(const char *path, bool purge)
return false;
}

+ new->fsid = init_path_fsid(new->path);
+ if (!new->fsid) {
+ free(new);
+ return false;
+ }
+
create_vdisks(new);
rb_insert(&md.root, new, rb, disk_cmp);
md.space += new->space;
@@ -904,3 +921,32 @@ uint32_t md_nr_disks(void)
{
return nr_online_disks();
}
+
+bool md_verify_disk(const char *path)
+{
+ struct statvfs fs;
+ struct disk *disk;
+ bool valid = true;
+
+ if (statvfs(path, &fs) < 0) {
+ sd_err("get disk %s stat failed %m", path);
+ return false;
+ }
+
+ sd_read_lock(&md.lock);
+ disk = path_to_disk(path);
+ if (!disk) {
+ /* already unpluged disk */
+ valid = false;
+ goto out;
+ }
+
+ if (fs.f_fsid != disk->fsid) {
+ /* mountpoint lost */
+ valid = false;
+ goto out;
+ }
+out:
+ sd_rw_unlock(&md.lock);
+ return valid;
+}
--
1.8.3.1