Commit 0500813f authored by Andreas Gruenbacher's avatar Andreas Gruenbacher Committed by Philipp Reisner
Browse files

drbd: Move conf_mutex from connection to resource 



Signed-off-by: default avatarAndreas Gruenbacher <agruen@linbit.com>
Signed-off-by: default avatarPhilipp Reisner <philipp.reisner@linbit.com>
parent 3ab706fe

drivers/block/drbd/drbd_int.h

+7 −8
Original line number Diff line number Diff line
@@ -518,7 +518,7 @@ struct drbd_backing_dev { 
	struct block_device *backing_bdev;

	struct block_device *md_bdev;

	struct drbd_md md;

	struct disk_conf *disk_conf; /* RCU, for updates: first_peer_device(device)->connection->conf_update */

	struct disk_conf *disk_conf; /* RCU, for updates: resource->conf_update */

	sector_t known_size; /* last known size of that backing device */

};
@@ -578,6 +578,8 @@ struct drbd_resource { 
	struct list_head connections;

	struct list_head resources;

	struct res_opts res_opts;

	struct mutex conf_update;	/* mutex for ready-copy-update of net_conf and disk_conf */

	spinlock_t req_lock;

};



struct drbd_connection {
@@ -594,7 +596,6 @@ struct drbd_connection { 


	unsigned long flags;

	struct net_conf *net_conf;	/* content protected by rcu */

	struct mutex conf_update;	/* mutex for ready-copy-update of net_conf and disk_conf */

	wait_queue_head_t ping_wait;	/* Woken upon reception of a ping, and a state change */



	struct sockaddr_storage my_addr;
@@ -608,8 +609,6 @@ struct drbd_connection { 
	unsigned long last_received;	/* in jiffies, either socket */

	unsigned int ko_count;



	spinlock_t req_lock;



	struct list_head transfer_log;	/* all requests not yet fully processed */



	struct crypto_hash *cram_hmac_tfm;
@@ -1595,9 +1594,9 @@ static inline void drbd_chk_io_error_(struct drbd_device *device, 
{

	if (error) {

		unsigned long flags;

		spin_lock_irqsave(&first_peer_device(device)->connection->req_lock, flags);

		spin_lock_irqsave(&device->resource->req_lock, flags);

		__drbd_chk_io_error_(device, forcedetach, where);

		spin_unlock_irqrestore(&first_peer_device(device)->connection->req_lock, flags);

		spin_unlock_irqrestore(&device->resource->req_lock, flags);

	}

}
@@ -2069,11 +2068,11 @@ static inline bool inc_ap_bio_cond(struct drbd_device *device) 
{

	bool rv = false;



	spin_lock_irq(&first_peer_device(device)->connection->req_lock);

	spin_lock_irq(&device->resource->req_lock);

	rv = may_inc_ap_bio(device);

	if (rv)

		atomic_inc(&device->ap_bio_cnt);

	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

	spin_unlock_irq(&device->resource->req_lock);



	return rv;

}

drivers/block/drbd/drbd_main.c

+18 −18
Original line number Diff line number Diff line
@@ -198,7 +198,7 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr, 
	int expect_epoch = 0;

	int expect_size = 0;



	spin_lock_irq(&connection->req_lock);

	spin_lock_irq(&connection->resource->req_lock);



	/* find oldest not yet barrier-acked write request,

	 * count writes in its epoch. */
@@ -255,12 +255,12 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr, 
			break;

		_req_mod(req, BARRIER_ACKED);

	}

	spin_unlock_irq(&connection->req_lock);

	spin_unlock_irq(&connection->resource->req_lock);



	return;



bail:

	spin_unlock_irq(&connection->req_lock);

	spin_unlock_irq(&connection->resource->req_lock);

	conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);

}
@@ -284,9 +284,9 @@ void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what) 


void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)

{

	spin_lock_irq(&connection->req_lock);

	spin_lock_irq(&connection->resource->req_lock);

	_tl_restart(connection, what);

	spin_unlock_irq(&connection->req_lock);

	spin_unlock_irq(&connection->resource->req_lock);

}



/**
@@ -311,7 +311,7 @@ void tl_abort_disk_io(struct drbd_device *device) 
	struct drbd_connection *connection = first_peer_device(device)->connection;

	struct drbd_request *req, *r;



	spin_lock_irq(&connection->req_lock);

	spin_lock_irq(&connection->resource->req_lock);

	list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {

		if (!(req->rq_state & RQ_LOCAL_PENDING))

			continue;
@@ -319,7 +319,7 @@ void tl_abort_disk_io(struct drbd_device *device) 
			continue;

		_req_mod(req, ABORT_DISK_IO);

	}

	spin_unlock_irq(&connection->req_lock);

	spin_unlock_irq(&connection->resource->req_lock);

}



static int drbd_thread_setup(void *arg)
@@ -1836,7 +1836,7 @@ static int drbd_open(struct block_device *bdev, fmode_t mode) 
	int rv = 0;



	mutex_lock(&drbd_main_mutex);

	spin_lock_irqsave(&first_peer_device(device)->connection->req_lock, flags);

	spin_lock_irqsave(&device->resource->req_lock, flags);

	/* to have a stable device->state.role

	 * and no race with updating open_cnt */
@@ -1849,7 +1849,7 @@ static int drbd_open(struct block_device *bdev, fmode_t mode) 


	if (!rv)

		device->open_cnt++;

	spin_unlock_irqrestore(&first_peer_device(device)->connection->req_lock, flags);

	spin_unlock_irqrestore(&device->resource->req_lock, flags);

	mutex_unlock(&drbd_main_mutex);



	return rv;
@@ -2546,6 +2546,8 @@ struct drbd_resource *drbd_create_resource(const char *name) 
	idr_init(&resource->devices);

	INIT_LIST_HEAD(&resource->connections);

	list_add_tail_rcu(&resource->resources, &drbd_resources);

	mutex_init(&resource->conf_update);

	spin_lock_init(&resource->req_lock);

	return resource;

}
@@ -2588,8 +2590,6 @@ struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts) 


	connection->cstate = C_STANDALONE;

	mutex_init(&connection->cstate_mutex);

	spin_lock_init(&connection->req_lock);

	mutex_init(&connection->conf_update);

	init_waitqueue_head(&connection->ping_wait);

	idr_init(&connection->peer_devices);
@@ -2720,7 +2720,7 @@ enum drbd_ret_code drbd_create_device(struct drbd_resource *resource, unsigned i 
	blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);

	blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);

	blk_queue_merge_bvec(q, drbd_merge_bvec);

	q->queue_lock = &connection->req_lock;

	q->queue_lock = &resource->req_lock;



	device->md_io_page = alloc_page(GFP_KERNEL);

	if (!device->md_io_page)
@@ -3281,14 +3281,14 @@ int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev) 


	rv = NO_ERROR;



	spin_lock_irq(&first_peer_device(device)->connection->req_lock);

	spin_lock_irq(&device->resource->req_lock);

	if (device->state.conn < C_CONNECTED) {

		unsigned int peer;

		peer = be32_to_cpu(buffer->la_peer_max_bio_size);

		peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);

		device->peer_max_bio_size = peer;

	}

	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

	spin_unlock_irq(&device->resource->req_lock);



 err:

	drbd_md_put_buffer(device);
@@ -3577,13 +3577,13 @@ void drbd_queue_bitmap_io(struct drbd_device *device, 
	device->bm_io_work.why = why;

	device->bm_io_work.flags = flags;



	spin_lock_irq(&first_peer_device(device)->connection->req_lock);

	spin_lock_irq(&device->resource->req_lock);

	set_bit(BITMAP_IO, &device->flags);

	if (atomic_read(&device->ap_bio_cnt) == 0) {

		if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))

			drbd_queue_work(&first_peer_device(device)->connection->sender_work, &device->bm_io_work.w);

	}

	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

	spin_unlock_irq(&device->resource->req_lock);

}



/**
@@ -3751,10 +3751,10 @@ int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i) 
	/* Indicate to wake up device->misc_wait on progress.  */

	i->waiting = true;

	prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);

	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

	spin_unlock_irq(&device->resource->req_lock);

	timeout = schedule_timeout(timeout);

	finish_wait(&device->misc_wait, &wait);

	spin_lock_irq(&first_peer_device(device)->connection->req_lock);

	spin_lock_irq(&device->resource->req_lock);

	if (!timeout || device->state.conn < C_CONNECTED)

		return -ETIMEDOUT;

	if (signal_pending(current))

drivers/block/drbd/drbd_nl.c

+25 −25
Original line number Diff line number Diff line
@@ -443,9 +443,9 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) 
		return false;

	}



	spin_lock_irq(&connection->req_lock);

	spin_lock_irq(&connection->resource->req_lock);

	connect_cnt = connection->connect_cnt;

	spin_unlock_irq(&connection->req_lock);

	spin_unlock_irq(&connection->resource->req_lock);



	fp = highest_fencing_policy(connection);

	switch (fp) {
@@ -510,7 +510,7 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) 
	   conn_request_state(connection, mask, val, CS_VERBOSE);

	   here, because we might were able to re-establish the connection in the

	   meantime. */

	spin_lock_irq(&connection->req_lock);

	spin_lock_irq(&connection->resource->req_lock);

	if (connection->cstate < C_WF_REPORT_PARAMS && !test_bit(STATE_SENT, &connection->flags)) {

		if (connection->connect_cnt != connect_cnt)

			/* In case the connection was established and droped
@@ -519,7 +519,7 @@ bool conn_try_outdate_peer(struct drbd_connection *connection) 
		else

			_conn_request_state(connection, mask, val, CS_VERBOSE);

	}

	spin_unlock_irq(&connection->req_lock);

	spin_unlock_irq(&connection->resource->req_lock);



	return conn_highest_pdsk(connection) <= D_OUTDATED;

}
@@ -654,11 +654,11 @@ drbd_set_role(struct drbd_device *device, enum drbd_role new_role, int force) 
			put_ldev(device);

		}

	} else {

		mutex_lock(&first_peer_device(device)->connection->conf_update);

		mutex_lock(&device->resource->conf_update);

		nc = first_peer_device(device)->connection->net_conf;

		if (nc)

			nc->discard_my_data = 0; /* without copy; single bit op is atomic */

		mutex_unlock(&first_peer_device(device)->connection->conf_update);

		mutex_unlock(&device->resource->conf_update);



		set_disk_ro(device->vdisk, false);

		if (get_ldev(device)) {
@@ -1188,10 +1188,10 @@ static void conn_reconfig_start(struct drbd_connection *connection) 
static void conn_reconfig_done(struct drbd_connection *connection)

{

	bool stop_threads;

	spin_lock_irq(&connection->req_lock);

	spin_lock_irq(&connection->resource->req_lock);

	stop_threads = conn_all_vols_unconf(connection) &&

		connection->cstate == C_STANDALONE;

	spin_unlock_irq(&connection->req_lock);

	spin_unlock_irq(&connection->resource->req_lock);

	if (stop_threads) {

		/* asender is implicitly stopped by receiver

		 * in conn_disconnect() */
@@ -1211,10 +1211,10 @@ static void drbd_suspend_al(struct drbd_device *device) 
	}



	drbd_al_shrink(device);

	spin_lock_irq(&first_peer_device(device)->connection->req_lock);

	spin_lock_irq(&device->resource->req_lock);

	if (device->state.conn < C_CONNECTED)

		s = !test_and_set_bit(AL_SUSPENDED, &device->flags);

	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

	spin_unlock_irq(&device->resource->req_lock);

	lc_unlock(device->act_log);



	if (s)
@@ -1285,7 +1285,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) 
		goto fail;

	}



	mutex_lock(&first_peer_device(device)->connection->conf_update);

	mutex_lock(&device->resource->conf_update);

	old_disk_conf = device->ldev->disk_conf;

	*new_disk_conf = *old_disk_conf;

	if (should_set_defaults(info))
@@ -1348,7 +1348,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) 
		rcu_assign_pointer(device->rs_plan_s, new_plan);

	}



	mutex_unlock(&first_peer_device(device)->connection->conf_update);

	mutex_unlock(&device->resource->conf_update);



	if (new_disk_conf->al_updates)

		device->ldev->md.flags &= ~MDF_AL_DISABLED;
@@ -1374,7 +1374,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) 
	goto success;



fail_unlock:

	mutex_unlock(&first_peer_device(device)->connection->conf_update);

	mutex_unlock(&device->resource->conf_update);

 fail:

	kfree(new_disk_conf);

	kfree(new_plan);
@@ -1724,7 +1724,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) 
	if (_drbd_bm_total_weight(device) == drbd_bm_bits(device))

		drbd_suspend_al(device); /* IO is still suspended here... */



	spin_lock_irq(&first_peer_device(device)->connection->req_lock);

	spin_lock_irq(&device->resource->req_lock);

	os = drbd_read_state(device);

	ns = os;

	/* If MDF_CONSISTENT is not set go into inconsistent state,
@@ -1776,7 +1776,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) 
	}



	rv = _drbd_set_state(device, ns, CS_VERBOSE, NULL);

	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

	spin_unlock_irq(&device->resource->req_lock);



	if (rv < SS_SUCCESS)

		goto force_diskless_dec;
@@ -2077,7 +2077,7 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) 
	conn_reconfig_start(connection);



	mutex_lock(&connection->data.mutex);

	mutex_lock(&connection->conf_update);

	mutex_lock(&connection->resource->conf_update);

	old_net_conf = connection->net_conf;



	if (!old_net_conf) {
@@ -2141,7 +2141,7 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) 
	crypto_free_hash(connection->cram_hmac_tfm);

	connection->cram_hmac_tfm = crypto.cram_hmac_tfm;



	mutex_unlock(&connection->conf_update);

	mutex_unlock(&connection->resource->conf_update);

	mutex_unlock(&connection->data.mutex);

	synchronize_rcu();

	kfree(old_net_conf);
@@ -2152,7 +2152,7 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) 
	goto done;



 fail:

	mutex_unlock(&connection->conf_update);

	mutex_unlock(&connection->resource->conf_update);

	mutex_unlock(&connection->data.mutex);

	free_crypto(&crypto);

	kfree(new_net_conf);
@@ -2243,11 +2243,11 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) 


	conn_flush_workqueue(connection);



	mutex_lock(&connection->conf_update);

	mutex_lock(&adm_ctx.resource->conf_update);

	old_net_conf = connection->net_conf;

	if (old_net_conf) {

		retcode = ERR_NET_CONFIGURED;

		mutex_unlock(&connection->conf_update);

		mutex_unlock(&adm_ctx.resource->conf_update);

		goto fail;

	}

	rcu_assign_pointer(connection->net_conf, new_net_conf);
@@ -2263,7 +2263,7 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) 
	connection->peer_addr_len = nla_len(adm_ctx.peer_addr);

	memcpy(&connection->peer_addr, nla_data(adm_ctx.peer_addr), connection->peer_addr_len);



	mutex_unlock(&connection->conf_update);

	mutex_unlock(&adm_ctx.resource->conf_update);



	rcu_read_lock();

	idr_for_each_entry(&connection->peer_devices, peer_device, i) {
@@ -2486,12 +2486,12 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info) 
		device->ldev->known_size = drbd_get_capacity(device->ldev->backing_bdev);



	if (new_disk_conf) {

		mutex_lock(&first_peer_device(device)->connection->conf_update);

		mutex_lock(&device->resource->conf_update);

		old_disk_conf = device->ldev->disk_conf;

		*new_disk_conf = *old_disk_conf;

		new_disk_conf->disk_size = (sector_t)rs.resize_size;

		rcu_assign_pointer(device->ldev->disk_conf, new_disk_conf);

		mutex_unlock(&first_peer_device(device)->connection->conf_update);

		mutex_unlock(&device->resource->conf_update);

		synchronize_rcu();

		kfree(old_disk_conf);

	}
@@ -3248,10 +3248,10 @@ int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info) 
			drbd_send_uuids_skip_initial_sync(device);

			_drbd_uuid_set(device, UI_BITMAP, 0);

			drbd_print_uuids(device, "cleared bitmap UUID");

			spin_lock_irq(&first_peer_device(device)->connection->req_lock);

			spin_lock_irq(&device->resource->req_lock);

			_drbd_set_state(_NS2(device, disk, D_UP_TO_DATE, pdsk, D_UP_TO_DATE),

					CS_VERBOSE, NULL);

			spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

			spin_unlock_irq(&device->resource->req_lock);

		}

	}

drivers/block/drbd/drbd_receiver.c

+58 −58

File changed.

Preview size limit exceeded, changes collapsed.

drivers/block/drbd/drbd_req.c

+9 −9
Original line number Diff line number Diff line
@@ -851,9 +851,9 @@ static void complete_conflicting_writes(struct drbd_request *req) 
			break;

		/* Indicate to wake up device->misc_wait on progress.  */

		i->waiting = true;

		spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

		spin_unlock_irq(&device->resource->req_lock);

		schedule();

		spin_lock_irq(&first_peer_device(device)->connection->req_lock);

		spin_lock_irq(&device->resource->req_lock);

	}

	finish_wait(&device->misc_wait, &wait);

}
@@ -1078,7 +1078,7 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request 
	struct bio_and_error m = { NULL, };

	bool no_remote = false;



	spin_lock_irq(&first_peer_device(device)->connection->req_lock);

	spin_lock_irq(&device->resource->req_lock);

	if (rw == WRITE) {

		/* This may temporarily give up the req_lock,

		 * but will re-aquire it before it returns here.
@@ -1140,9 +1140,9 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request 
		/* needs to be marked within the same spinlock */

		_req_mod(req, TO_BE_SUBMITTED);

		/* but we need to give up the spinlock to submit */

		spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

		spin_unlock_irq(&device->resource->req_lock);

		drbd_submit_req_private_bio(req);

		spin_lock_irq(&first_peer_device(device)->connection->req_lock);

		spin_lock_irq(&device->resource->req_lock);

	} else if (no_remote) {

nodata:

		if (__ratelimit(&drbd_ratelimit_state))
@@ -1155,7 +1155,7 @@ static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request 
out:

	if (drbd_req_put_completion_ref(req, &m, 1))

		kref_put(&req->kref, drbd_req_destroy);

	spin_unlock_irq(&first_peer_device(device)->connection->req_lock);

	spin_unlock_irq(&device->resource->req_lock);



	if (m.bio)

		complete_master_bio(device, &m);
@@ -1360,10 +1360,10 @@ void request_timer_fn(unsigned long data) 


	now = jiffies;



	spin_lock_irq(&connection->req_lock);

	spin_lock_irq(&device->resource->req_lock);

	req = find_oldest_request(connection);

	if (!req) {

		spin_unlock_irq(&connection->req_lock);

		spin_unlock_irq(&device->resource->req_lock);

		mod_timer(&device->request_timer, now + et);

		return;

	}
@@ -1397,6 +1397,6 @@ void request_timer_fn(unsigned long data) 
		__drbd_chk_io_error(device, DRBD_FORCE_DETACH);

	}

	nt = (time_after(now, req->start_time + et) ? now : req->start_time) + et;

	spin_unlock_irq(&connection->req_lock);

	spin_unlock_irq(&connection->resource->req_lock);

	mod_timer(&device->request_timer, nt);

}