bcachefs: Hacky fixes for device removal

The device remove test was sporadically failing, because we hadn't
finished dropping btree sector counts for the device when
bch2_replicas_gc2() was called - mainly due to in flight journal writes.
We don't yet have a good mechanism for flushing the counts that
correspend to open journal entries yet.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

1 files changed, 26 insertions, 17 deletions

diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c
index cd02e5a5f305..586636a4c204 100644
--- a/fs/bcachefs/super.c
+++ b/fs/bcachefs/super.c
@@ -1381,7 +1381,11 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags)
 
 	mutex_lock(&c->state_lock);
 
-	percpu_ref_put(&ca->ref); /* XXX */
+	/*
+	 * We consume a reference to ca->ref, regardless of whether we succeed
+	 * or fail:
+	 */
+	percpu_ref_put(&ca->ref);
 
 	if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_FAILED, flags)) {
 		bch_err(ca, "Cannot remove without losing data");
@@ -1390,11 +1394,6 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags)
 
 	__bch2_dev_read_only(c, ca);
 
-	/*
-	 * XXX: verify that dev_idx is really not in use anymore, anywhere
-	 *
-	 * flag_data_bad() does not check btree pointers
-	 */
 	ret = bch2_dev_data_drop(c, ca->dev_idx, flags);
 	if (ret) {
 		bch_err(ca, "Remove failed: error %i dropping data", ret);
@@ -1407,17 +1406,6 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags)
 		goto err;
 	}
 
-	data = bch2_dev_has_data(c, ca);
-	if (data) {
-		char data_has_str[100];
-
-		bch2_flags_to_text(&PBUF(data_has_str),
-				   bch2_data_types, data);
-		bch_err(ca, "Remove failed, still has data (%s)", data_has_str);
-		ret = -EBUSY;
-		goto err;
-	}
-
 	ret = bch2_btree_delete_range(c, BTREE_ID_ALLOC,
 				      POS(ca->dev_idx, 0),
 				      POS(ca->dev_idx + 1, 0),
@@ -1432,12 +1420,33 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags)
 	 * (overwritten) keys that point to the device we're removing:
 	 */
 	bch2_journal_flush_all_pins(&c->journal);
+	/*
+	 * hack to ensure bch2_replicas_gc2() clears out entries to this device
+	 */
+	bch2_journal_meta(&c->journal);
 	ret = bch2_journal_error(&c->journal);
 	if (ret) {
 		bch_err(ca, "Remove failed, journal error");
 		goto err;
 	}
 
+	ret = bch2_replicas_gc2(c);
+	if (ret) {
+		bch_err(ca, "Remove failed: error %i from replicas gc", ret);
+		goto err;
+	}
+
+	data = bch2_dev_has_data(c, ca);
+	if (data) {
+		char data_has_str[100];
+
+		bch2_flags_to_text(&PBUF(data_has_str),
+				   bch2_data_types, data);
+		bch_err(ca, "Remove failed, still has data (%s)", data_has_str);
+		ret = -EBUSY;
+		goto err;
+	}
+
 	__bch2_dev_offline(c, ca);
 
 	mutex_lock(&c->sb_lock);