Simple Metrics

Terms defined: Little's Law

Measuring Delay

@dataclass_json
@dataclass
class Params:
    t_job_arrival: float = 2.0
    t_job_mean: float = 0.5
    t_job_std: float = 0.6
    t_sim: float = 10
class Job:
    SAVE_KEYS = ["t_create", "t_start", "t_complete"]
    _next_id = count()
    _all = []

    @classmethod
    def reset(cls):
        cls._next_id = count()
        cls._all = []

    def __init__(self, sim):
        Job._all.append(self)
        self.id = next(Job._next_id)
        self.duration = sim.rand_job_duration()
        self.t_create = sim.env.now
        self.t_start = None
        self.t_complete = None

    def json(self):
        return {key: util.rnd(self, key) for key in self.SAVE_KEYS}
@dataclass
class Simulation(Environment):
    def __init__(self):
        super().__init__()
        self.params = Params()
        self.queue = Store(self)

    def simulate(self):
        Job.reset()
        self.queue = Store(self.env)
        self.env.process(manager(self))
        self.env.process(coder(self))
        self.env.run(until=self.params.t_sim)
def manager(sim):
    while True:
        job = Job(sim=sim)
        yield sim.queue.put(job)
        yield sim.env.timeout(sim.rand_job_arrival())


def coder(sim):
    while True:
        job = yield sim.queue.get()
        job.t_start = sim.env.now
        yield sim.env.timeout(job.duration)
        job.t_complete = sim.env.now
## jobs
shape: (8, 9)
┌──────────┬─────────┬────────────┬─────┬───┬───────────────┬────────────┬───────────┬───────┐
│ t_create ┆ t_start ┆ t_complete ┆ id  ┆ … ┆ t_job_arrival ┆ t_job_mean ┆ t_job_std ┆ t_sim │
│ ---      ┆ ---     ┆ ---        ┆ --- ┆   ┆ ---           ┆ ---        ┆ ---       ┆ ---   │
│ f64      ┆ f64     ┆ f64        ┆ i32 ┆   ┆ f64           ┆ f64        ┆ f64       ┆ i32   │
╞══════════╪═════════╪════════════╪═════╪═══╪═══════════════╪════════════╪═══════════╪═══════╡
│ 0.0      ┆ 0.0     ┆ 0.68       ┆ 0   ┆ … ┆ 2.0           ┆ 0.5        ┆ 0.6       ┆ 10    │
│ 3.12     ┆ 3.12    ┆ 3.99       ┆ 0   ┆ … ┆ 2.0           ┆ 0.5        ┆ 0.6       ┆ 10    │
│ 4.69     ┆ 4.69    ┆ 8.33       ┆ 0   ┆ … ┆ 2.0           ┆ 0.5        ┆ 0.6       ┆ 10    │
│ 5.25     ┆ 8.33    ┆ 9.15       ┆ 0   ┆ … ┆ 2.0           ┆ 0.5        ┆ 0.6       ┆ 10    │
│ 6.87     ┆ 9.15    ┆ null       ┆ 0   ┆ … ┆ 2.0           ┆ 0.5        ┆ 0.6       ┆ 10    │
│ 8.84     ┆ null    ┆ null       ┆ 0   ┆ … ┆ 2.0           ┆ 0.5        ┆ 0.6       ┆ 10    │
│ 9.0      ┆ null    ┆ null       ┆ 0   ┆ … ┆ 2.0           ┆ 0.5        ┆ 0.6       ┆ 10    │
│ 9.37     ┆ null    ┆ null       ┆ 0   ┆ … ┆ 2.0           ┆ 0.5        ┆ 0.6       ┆ 10    │
└──────────┴─────────┴────────────┴─────┴───┴───────────────┴────────────┴───────────┴───────┘
if __name__ == "__main__":
    args, results = util.run(Params, Simulation)
    jobs = util.as_frames(results)["jobs"]
    jobs = jobs \
        .filter(pl.col("t_start").is_not_null()) \
        .sort("t_create") \
        .with_columns((pl.col("t_start") - pl.col("t_create")).alias("delay"))
    fig = px.line(jobs, x="t_start", y="delay", facet_col="t_sim")
job delays vs. time
Figure 1: Job delays vs. time

Four Metrics

class Recorder:
    _next_id = defaultdict(count)
    _all = defaultdict(list)

    @staticmethod
    def reset():
        Recorder._next_id = defaultdict(count)
        Recorder._all = defaultdict(list)

    def __init__(self, sim):
        cls = self.__class__
        self.id = next(self._next_id[cls])
        self._all[cls].append(self)
        self.sim = sim

    def json(self):
        return {key: util.rnd(self, key) for key in self.SAVE_KEYS}
class Manager(Recorder):
    def run(self):
        while True:
            job = Job(sim=self.sim)
            yield self.sim.queue.put(job)
            yield self.sim.timeout(self.sim.rand_job_arrival())
class Coder(Recorder):
    SAVE_KEYS = ["t_work"]

    def __init__(self, sim):
        super().__init__(sim)
        self.t_work = 0

    def run(self):
        while True:
            job = yield self.sim.queue.get()
            job.t_start = self.sim.now
            yield self.sim.timeout(job.duration)
            job.t_complete = self.sim.now
            self.t_work += job.t_complete - job.t_start
class Monitor(Recorder):
    def run(self):
        while True:
            self.sim.lengths.append(
                {"time": self.sim.now, "length": len(self.sim.queue.items)}
            )
            yield self.sim.timeout(self.sim.params.t_monitor)
class Simulation:
    # …as before…
    def run(self):
        Recorder.reset()
        self.queue = Store(self.env)
        self.env.process(Manager(self).run())
        self.env.process(Coder(self).run())
        self.env.process(Monitor(self).run())
        self.env.run(until=self.params.t_sim)
class Simulation:
    # …as before…
    def result(self):
        return {
            "jobs": [job.json() for job in Recorder._all[Job]],
            "coders": [coder.json() for coder in Recorder._all[Coder]],
            "lengths": self.lengths,
        }
job delays vs. time
Figure 2: Job delays vs. time
backlog vs. time
Figure 3: Backlog vs. time
Table 1: Throughput
id t_sim num_jobs throughput
0 200 96 0.48
1 200 92 0.46
2 1000 489 0.49
3 1000 492 0.49
Table 2: Utilization
id t_sim total_work utilization
0 200 172.94 0.86
1 200 175.23 0.88
2 1000 947.52 0.95
3 1000 931.92 0.93
  1. Backlog and delay track each other pretty closely, so we only need to measure one or the other.
  2. Throughput stabilizes right away. Utilization takes a little longer, but even then the change is pretty small as we increase the length of the simulation.

Imagine you're the manager in the fourth scenario. You might panic as backlog starts to rise, not realizing that it's just random variation.

Varying Arrival Rate

backlog vs. time with varying job arrival rates
Figure 4: backlog vs. time with varying job arrival rates
backlog vs. time with narrower range of job arrival rates
Figure 5: backlog vs. time with narrower range of job arrival rates

Little's Law