Commit 08420cf7 authored by Jens Axboe's avatar Jens Axboe
Browse files

block: add blk_time_get_ns() and blk_time_get() helpers 



Convert any user of ktime_get_ns() to use blk_time_get_ns(), and
ktime_get() to blk_time_get(), so we have a unified API for querying the
current time in nanoseconds or as ktime.

No functional changes intended, this patch just wraps ktime_get_ns()
and ktime_get() with a block helper.

Reviewed-by: default avatarJohannes Thumshirn <johannes.thumshirn@wdc.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent c4e47bbb

block/bfq-cgroup.c

+7 −7
Original line number Diff line number Diff line
@@ -127,7 +127,7 @@ static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats) 
	if (!bfqg_stats_waiting(stats))

		return;



	now = ktime_get_ns();

	now = blk_time_get_ns();

	if (now > stats->start_group_wait_time)

		bfq_stat_add(&stats->group_wait_time,

			      now - stats->start_group_wait_time);
@@ -144,7 +144,7 @@ static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg, 
		return;

	if (bfqg == curr_bfqg)

		return;

	stats->start_group_wait_time = ktime_get_ns();

	stats->start_group_wait_time = blk_time_get_ns();

	bfqg_stats_mark_waiting(stats);

}
@@ -156,7 +156,7 @@ static void bfqg_stats_end_empty_time(struct bfqg_stats *stats) 
	if (!bfqg_stats_empty(stats))

		return;



	now = ktime_get_ns();

	now = blk_time_get_ns();

	if (now > stats->start_empty_time)

		bfq_stat_add(&stats->empty_time,

			      now - stats->start_empty_time);
@@ -183,7 +183,7 @@ void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) 
	if (bfqg_stats_empty(stats))

		return;



	stats->start_empty_time = ktime_get_ns();

	stats->start_empty_time = blk_time_get_ns();

	bfqg_stats_mark_empty(stats);

}
@@ -192,7 +192,7 @@ void bfqg_stats_update_idle_time(struct bfq_group *bfqg) 
	struct bfqg_stats *stats = &bfqg->stats;



	if (bfqg_stats_idling(stats)) {

		u64 now = ktime_get_ns();

		u64 now = blk_time_get_ns();



		if (now > stats->start_idle_time)

			bfq_stat_add(&stats->idle_time,
@@ -205,7 +205,7 @@ void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) 
{

	struct bfqg_stats *stats = &bfqg->stats;



	stats->start_idle_time = ktime_get_ns();

	stats->start_idle_time = blk_time_get_ns();

	bfqg_stats_mark_idling(stats);

}
@@ -242,7 +242,7 @@ void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, 
				  u64 io_start_time_ns, blk_opf_t opf)

{

	struct bfqg_stats *stats = &bfqg->stats;

	u64 now = ktime_get_ns();

	u64 now = blk_time_get_ns();



	if (now > io_start_time_ns)

		blkg_rwstat_add(&stats->service_time, opf,

block/bfq-iosched.c

+14 −14
Original line number Diff line number Diff line
@@ -1005,7 +1005,7 @@ static struct request *bfq_check_fifo(struct bfq_queue *bfqq, 


	rq = rq_entry_fifo(bfqq->fifo.next);



	if (rq == last || ktime_get_ns() < rq->fifo_time)

	if (rq == last || blk_time_get_ns() < rq->fifo_time)

		return NULL;



	bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq);
@@ -1829,7 +1829,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, 
		 * bfq_bfqq_update_budg_for_activation for

		 * details on the usage of the next variable.

		 */

		arrived_in_time =  ktime_get_ns() <=

		arrived_in_time =  blk_time_get_ns() <=

			bfqq->ttime.last_end_request +

			bfqd->bfq_slice_idle * 3;

	unsigned int act_idx = bfq_actuator_index(bfqd, rq->bio);
@@ -2208,7 +2208,7 @@ static void bfq_add_request(struct request *rq) 
	struct request *next_rq, *prev;

	unsigned int old_wr_coeff = bfqq->wr_coeff;

	bool interactive = false;

	u64 now_ns = ktime_get_ns();

	u64 now_ns = blk_time_get_ns();



	bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq));

	bfqq->queued[rq_is_sync(rq)]++;
@@ -2262,7 +2262,7 @@ static void bfq_add_request(struct request *rq) 
		      bfqd->rqs_injected && bfqd->tot_rq_in_driver > 0)) &&

		    time_is_before_eq_jiffies(bfqq->decrease_time_jif +

					      msecs_to_jiffies(10))) {

			bfqd->last_empty_occupied_ns = ktime_get_ns();

			bfqd->last_empty_occupied_ns = blk_time_get_ns();

			/*

			 * Start the state machine for measuring the

			 * total service time of rq: setting
@@ -3294,7 +3294,7 @@ static void bfq_set_budget_timeout(struct bfq_data *bfqd, 
	else

		timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight;



	bfqd->last_budget_start = ktime_get();

	bfqd->last_budget_start = blk_time_get();



	bfqq->budget_timeout = jiffies +

		bfqd->bfq_timeout * timeout_coeff;
@@ -3394,7 +3394,7 @@ static void bfq_arm_slice_timer(struct bfq_data *bfqd) 
	else if (bfqq->wr_coeff > 1)

		sl = max_t(u32, sl, 20ULL * NSEC_PER_MSEC);



	bfqd->last_idling_start = ktime_get();

	bfqd->last_idling_start = blk_time_get();

	bfqd->last_idling_start_jiffies = jiffies;



	hrtimer_start(&bfqd->idle_slice_timer, ns_to_ktime(sl),
@@ -3433,7 +3433,7 @@ static void bfq_reset_rate_computation(struct bfq_data *bfqd, 
				       struct request *rq)

{

	if (rq != NULL) { /* new rq dispatch now, reset accordingly */

		bfqd->last_dispatch = bfqd->first_dispatch = ktime_get_ns();

		bfqd->last_dispatch = bfqd->first_dispatch = blk_time_get_ns();

		bfqd->peak_rate_samples = 1;

		bfqd->sequential_samples = 0;

		bfqd->tot_sectors_dispatched = bfqd->last_rq_max_size =
@@ -3590,7 +3590,7 @@ static void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) 
 */

static void bfq_update_peak_rate(struct bfq_data *bfqd, struct request *rq)

{

	u64 now_ns = ktime_get_ns();

	u64 now_ns = blk_time_get_ns();



	if (bfqd->peak_rate_samples == 0) { /* first dispatch */

		bfq_log(bfqd, "update_peak_rate: goto reset, samples %d",
@@ -4162,7 +4162,7 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, 
	if (compensate)

		delta_ktime = bfqd->last_idling_start;

	else

		delta_ktime = ktime_get();

		delta_ktime = blk_time_get();

	delta_ktime = ktime_sub(delta_ktime, bfqd->last_budget_start);

	delta_usecs = ktime_to_us(delta_ktime);
@@ -5591,7 +5591,7 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, 
			  struct bfq_io_cq *bic, pid_t pid, int is_sync,

			  unsigned int act_idx)

{

	u64 now_ns = ktime_get_ns();

	u64 now_ns = blk_time_get_ns();



	bfqq->actuator_idx = act_idx;

	RB_CLEAR_NODE(&bfqq->entity.rb_node);
@@ -5903,7 +5903,7 @@ static void bfq_update_io_thinktime(struct bfq_data *bfqd, 
	 */

	if (bfqq->dispatched || bfq_bfqq_busy(bfqq))

		return;

	elapsed = ktime_get_ns() - bfqq->ttime.last_end_request;

	elapsed = blk_time_get_ns() - bfqq->ttime.last_end_request;

	elapsed = min_t(u64, elapsed, 2ULL * bfqd->bfq_slice_idle);



	ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8;
@@ -6194,7 +6194,7 @@ static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq) 
	bfq_add_request(rq);

	idle_timer_disabled = waiting && !bfq_bfqq_wait_request(bfqq);



	rq->fifo_time = ktime_get_ns() + bfqd->bfq_fifo_expire[rq_is_sync(rq)];

	rq->fifo_time = blk_time_get_ns() + bfqd->bfq_fifo_expire[rq_is_sync(rq)];

	list_add_tail(&rq->queuelist, &bfqq->fifo);



	bfq_rq_enqueued(bfqd, bfqq, rq);
@@ -6370,7 +6370,7 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) 
		bfq_weights_tree_remove(bfqq);

	}



	now_ns = ktime_get_ns();

	now_ns = blk_time_get_ns();



	bfqq->ttime.last_end_request = now_ns;
@@ -6585,7 +6585,7 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) 
static void bfq_update_inject_limit(struct bfq_data *bfqd,

				    struct bfq_queue *bfqq)

{

	u64 tot_time_ns = ktime_get_ns() - bfqd->last_empty_occupied_ns;

	u64 tot_time_ns = blk_time_get_ns() - bfqd->last_empty_occupied_ns;

	unsigned int old_limit = bfqq->inject_limit;



	if (bfqq->last_serv_time_ns > 0 && bfqd->rqs_injected) {

block/blk-cgroup.c

+1 −1
Original line number Diff line number Diff line
@@ -1846,7 +1846,7 @@ static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay) 
{

	unsigned long pflags;

	bool clamp;

	u64 now = ktime_to_ns(ktime_get());

	u64 now = blk_time_get_ns();

	u64 exp;

	u64 delay_nsec = 0;

	int tok;

block/blk-flush.c

+1 −1
Original line number Diff line number Diff line
@@ -143,7 +143,7 @@ static void blk_account_io_flush(struct request *rq) 
	part_stat_lock();

	part_stat_inc(part, ios[STAT_FLUSH]);

	part_stat_add(part, nsecs[STAT_FLUSH],

		      ktime_get_ns() - rq->start_time_ns);

		      blk_time_get_ns() - rq->start_time_ns);

	part_stat_unlock();

}

block/blk-iocost.c

+4 −4
Original line number Diff line number Diff line
@@ -829,7 +829,7 @@ static int ioc_autop_idx(struct ioc *ioc, struct gendisk *disk) 


	/* step up/down based on the vrate */

	vrate_pct = div64_u64(ioc->vtime_base_rate * 100, VTIME_PER_USEC);

	now_ns = ktime_get_ns();

	now_ns = blk_time_get_ns();



	if (p->too_fast_vrate_pct && p->too_fast_vrate_pct <= vrate_pct) {

		if (!ioc->autop_too_fast_at)
@@ -1044,7 +1044,7 @@ static void ioc_now(struct ioc *ioc, struct ioc_now *now) 
	unsigned seq;

	u64 vrate;



	now->now_ns = ktime_get();

	now->now_ns = blk_time_get_ns();

	now->now = ktime_to_us(now->now_ns);

	vrate = atomic64_read(&ioc->vtime_rate);
@@ -2810,7 +2810,7 @@ static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq) 
		return;

	}



	on_q_ns = ktime_get_ns() - rq->alloc_time_ns;

	on_q_ns = blk_time_get_ns() - rq->alloc_time_ns;

	rq_wait_ns = rq->start_time_ns - rq->alloc_time_ns;

	size_nsec = div64_u64(calc_size_vtime_cost(rq, ioc), VTIME_PER_NSEC);
@@ -2893,7 +2893,7 @@ static int blk_iocost_init(struct gendisk *disk) 
	ioc->vtime_base_rate = VTIME_PER_USEC;

	atomic64_set(&ioc->vtime_rate, VTIME_PER_USEC);

	seqcount_spinlock_init(&ioc->period_seqcount, &ioc->lock);

	ioc->period_at = ktime_to_us(ktime_get());

	ioc->period_at = ktime_to_us(blk_time_get());

	atomic64_set(&ioc->cur_period, 0);

	atomic_set(&ioc->hweight_gen, 0);