// tsccheck: watch TSC rate (vs the RTC crystal) and cross-core sync (vs CPU0).
// build: gcc -O2 -Wall -pthread -o tsccheck tsccheck.c       (x86-64 Linux)
// usage: sudo ./tsccheck [interval_s] [samples] [kernel_mhz]   (root for /dev/rtc0)
// kernel_mhz: journalctl -k -b | grep 'Refined TSC'
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <linux/rtc.h>
#include <pthread.h>
#include <sched.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <time.h>
#include <unistd.h>

#define ITERS 3000

static inline uint64_t rdtsc_o(void)
{
	unsigned lo, hi;
	__asm__ volatile("lfence; rdtsc; lfence" : "=a"(lo), "=d"(hi) :: "memory");
	return ((uint64_t)hi << 32) | lo;
}

static void pin(int cpu)
{
	cpu_set_t s;
	CPU_ZERO(&s);
	CPU_SET(cpu, &s);
	if (pthread_setaffinity_np(pthread_self(), sizeof s, &s)) {
		fprintf(stderr, "cannot pin to cpu %d\n", cpu);
		exit(1);
	}
}

static struct {
	_Atomic int seq;
	_Atomic uint64_t t0, t1;
} __attribute__((aligned(64))) pp;

static void *ap_thread(void *arg)
{
	pin((int)(long)arg);
	for (int i = 0; i < ITERS; i++) {
		while (atomic_load(&pp.seq) != 2 * i + 1)
			;
		atomic_store(&pp.t1, rdtsc_o());
		atomic_store(&pp.seq, 2 * i + 2);
	}
	return NULL;
}

struct sync_res { int64_t offset; uint64_t rtt; int warps; };

/* offset = AP tsc - BSP tsc, from the lowest-rtt round trip; warp = causality broken */
static struct sync_res measure(int cpu)
{
	struct sync_res r = { 0, UINT64_MAX, 0 };
	pthread_t th;
	atomic_store(&pp.seq, 0);
	pthread_create(&th, NULL, ap_thread, (void *)(long)cpu);
	for (int i = 0; i < ITERS; i++) {
		uint64_t t0 = rdtsc_o();
		atomic_store(&pp.t0, t0);
		atomic_store(&pp.seq, 2 * i + 1);
		while (atomic_load(&pp.seq) != 2 * i + 2)
			;
		uint64_t t2 = rdtsc_o();
		uint64_t t1 = atomic_load(&pp.t1);
		if (t1 < t0 || t2 < t1)
			r.warps++;
		if (i > 100 && t2 - t0 < r.rtt) {
			r.rtt = t2 - t0;
			r.offset = (int64_t)(t1 - (t0 + (t2 - t0) / 2));
		}
	}
	pthread_join(th, NULL);
	return r;
}

static void clocksource(char *buf, size_t n)
{
	FILE *f = fopen("/sys/devices/system/clocksource/clocksource0/current_clocksource", "r");
	if (!f || !fgets(buf, n, f))
		snprintf(buf, n, "?");
	if (f)
		fclose(f);
	buf[strcspn(buf, "\n")] = 0;
}

int main(int argc, char **argv)
{
	int interval = argc > 1 ? atoi(argv[1]) : 15;
	int samples = argc > 2 ? atoi(argv[2]) : 20;
	double kmhz = argc > 3 ? atof(argv[3]) : 2096.073;	/* the E595's refined figure */
	int ncpu = sysconf(_SC_NPROCESSORS_ONLN);
	unsigned long d;
	char cs[64];

	setvbuf(stdout, NULL, _IOLBF, 0);
	pin(0);
	int fd = open("/dev/rtc0", O_RDONLY);
	if (fd < 0 || ioctl(fd, RTC_UIE_ON, 0) < 0) {
		perror("/dev/rtc0");
		return 1;
	}
	read(fd, &d, sizeof d);	/* align to an RTC second edge */
	uint64_t start = rdtsc_o(), prev = start;

	double mn = 1e9, mx = 0;
	int64_t worst_off = 0;
	int total_warps = 0, cs_bad = 0;

	printf("reference: RTC 1 Hz edges; kernel refined %.3f MHz; %d CPUs; %d x %d s\n",
	       kmhz, ncpu, samples, interval);
	printf("  time   window MHz  vs kernel   cumulative   max|off|  warps  rtt  clksrc   offsets cpu1..%d (cycles)\n", ncpu - 1);
	for (int s = 1; s <= samples; s++) {
		for (int k = 0; k < interval; k++)
			if (read(fd, &d, sizeof d) < 0) {
				perror("rtc read");
				return 1;
			}
		uint64_t now = rdtsc_o();
		double mhz = (now - prev) / (interval * 1e6);
		double cum = (now - start) / ((double)interval * s * 1e6);
		prev = now;
		if (mhz < mn) mn = mhz;
		if (mhz > mx) mx = mhz;

		char offs[256] = "";
		int64_t maxo = 0;
		int warps = 0;
		uint64_t maxrtt = 0;
		for (int c = 1; c < ncpu; c++) {
			struct sync_res r = measure(c);
			char b[24];
			snprintf(b, sizeof b, " %+lld", (long long)r.offset);
			strcat(offs, b);
			if (llabs(r.offset) > llabs(maxo)) maxo = r.offset;
			if (r.rtt > maxrtt) maxrtt = r.rtt;
			warps += r.warps;
		}
		if (llabs(maxo) > llabs(worst_off)) worst_off = maxo;
		total_warps += warps;
		clocksource(cs, sizeof cs);
		if (strcmp(cs, "tsc")) cs_bad++;
		printf("%5ds   %9.4f   %+7.1f ppm  %9.4f   %6lld   %4d  %4llu  %-6s %s\n",
		       s * interval, mhz, (mhz / kmhz - 1) * 1e6, cum,
		       (long long)maxo, warps, (unsigned long long)maxrtt, cs, offs);
	}
	ioctl(fd, RTC_UIE_OFF, 0);
	printf("SUMMARY: window rate %.4f..%.4f MHz (spread %.1f ppm), worst offset %lld cycles (%.1f ns), "
	       "warps %d, samples off tsc %d\n",
	       mn, mx, (mx - mn) / mn * 1e6, (long long)worst_off, worst_off / kmhz * 1e3,
	       total_warps, cs_bad);
	return 0;
}