[{"data":1,"prerenderedAt":259},["ShallowReactive",2],{"blog-reproducible-gpu-benchmarks":3},{"id":4,"title":5,"algorithms":6,"author":7,"body":8,"canonical_url":222,"cover_alt":223,"cover_image":223,"demo_url":223,"description":14,"extension":224,"featured":62,"kind":225,"languages":226,"learning_objectives":227,"limitations":231,"meta":233,"modified":234,"navigation":62,"path":235,"projects":236,"published":238,"relations":239,"seo":243,"slug":244,"source_url":223,"stem":245,"summary":246,"tags":247,"technologies":251,"topics":254,"version":257,"__hash__":258},"blog\u002Fblog\u002Freproducible-gpu-benchmarks.md","A benchmark is an experimental record",[],"Lucas Aruodore Adomi",{"type":9,"value":10,"toc":215},"minimark",[11,15,20,23,26,30,33,117,120,124,127,194,198,201,204,208,211],[12,13,14],"p",{},"A benchmark result is not a number. It is a number attached to an environment, a protocol, raw observations, and a question. Remove any one of those and the result becomes difficult to interpret. Remove several and it becomes decoration.",[16,17,19],"h2",{"id":18},"begin-with-the-decision","Begin with the decision",[12,21,22],{},"“Which implementation is faster?” is incomplete. A useful benchmark names the workload and the decision it supports. Latency for batch size one, throughput under saturation, memory pressure during training, and energy per inference are different experiments.",[12,24,25],{},"The metric follows the decision. It should not be chosen because it produces the largest ratio.",[16,27,29],{"id":28},"synchronization-changes-the-measurement","Synchronization changes the measurement",[12,31,32],{},"CUDA kernels are dispatched asynchronously. Timing only the Python call often measures queue submission rather than device execution.",[34,35,40],"pre",{"className":36,"code":37,"language":38,"meta":39,"style":39},"language-python shiki shiki-themes github-light github-dark","import time\nimport torch\n\ndef timed_run(operation, samples=30):\n    observations = []\n    for _ in range(samples):\n        torch.cuda.synchronize()\n        started = time.perf_counter_ns()\n        operation()\n        torch.cuda.synchronize()\n        observations.append(time.perf_counter_ns() - started)\n    return observations\n","python","",[41,42,43,51,57,64,70,76,82,88,94,100,105,111],"code",{"__ignoreMap":39},[44,45,48],"span",{"class":46,"line":47},"line",1,[44,49,50],{},"import time\n",[44,52,54],{"class":46,"line":53},2,[44,55,56],{},"import torch\n",[44,58,60],{"class":46,"line":59},3,[44,61,63],{"emptyLinePlaceholder":62},true,"\n",[44,65,67],{"class":46,"line":66},4,[44,68,69],{},"def timed_run(operation, samples=30):\n",[44,71,73],{"class":46,"line":72},5,[44,74,75],{},"    observations = []\n",[44,77,79],{"class":46,"line":78},6,[44,80,81],{},"    for _ in range(samples):\n",[44,83,85],{"class":46,"line":84},7,[44,86,87],{},"        torch.cuda.synchronize()\n",[44,89,91],{"class":46,"line":90},8,[44,92,93],{},"        started = time.perf_counter_ns()\n",[44,95,97],{"class":46,"line":96},9,[44,98,99],{},"        operation()\n",[44,101,103],{"class":46,"line":102},10,[44,104,87],{},[44,106,108],{"class":46,"line":107},11,[44,109,110],{},"        observations.append(time.perf_counter_ns() - started)\n",[44,112,114],{"class":46,"line":113},12,[44,115,116],{},"    return observations\n",[12,118,119],{},"Synchronization is not free, so the measured region must be documented. Device events are preferable when the question is strictly kernel execution. End-to-end wall time is preferable when host overhead is part of the user experience.",[16,121,123],{"id":122},"keep-the-distribution","Keep the distribution",[12,125,126],{},"Thirty observations contain information that their mean discards. Report the median, an interval, and the raw values. A violin or empirical cumulative distribution can reveal multimodality caused by thermal behavior, compilation, or background work.",[128,129,130,146],"table",{},[131,132,133],"thead",{},[134,135,136,140,143],"tr",{},[137,138,139],"th",{},"Field",[137,141,142],{},"Example value",[137,144,145],{},"Why it matters",[147,148,149,161,172,183],"tbody",{},[134,150,151,155,158],{},[152,153,154],"td",{},"Warmups",[152,156,157],{},"10",[152,159,160],{},"Excludes compilation and initial allocation",[134,162,163,166,169],{},[152,164,165],{},"Samples",[152,167,168],{},"30",[152,170,171],{},"Exposes run-to-run variation",[134,173,174,177,180],{},[152,175,176],{},"Statistic",[152,178,179],{},"Median",[152,181,182],{},"Resists a small number of long stalls",[134,184,185,188,191],{},[152,186,187],{},"Interval",[152,189,190],{},"Bootstrap 95%",[152,192,193],{},"Communicates estimation uncertainty",[16,195,197],{"id":196},"preserve-the-environment","Preserve the environment",[12,199,200],{},"Record the device model, driver, runtime, dependency lock hash, operating system, power policy, precision, shapes, dataset version, and full command. Reference source by immutable commit. Raw result files should use a run identifier and checksum.",[12,202,203],{},"A future result should be compared automatically only when the methodology declares the runs compatible. Otherwise, show both records and explain the environmental differences.",[16,205,207],{"id":206},"the-honest-conclusion","The honest conclusion",[12,209,210],{},"Benchmarks are valuable when they narrow a decision. “Implementation A was faster under this protocol” is stronger than an unsupported universal claim. The caveat does not weaken the work. It tells the reader where the evidence applies.",[212,213,214],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":39,"searchDepth":59,"depth":59,"links":216},[217,218,219,220,221],{"id":18,"depth":53,"text":19},{"id":28,"depth":53,"text":29},{"id":122,"depth":53,"text":123},{"id":196,"depth":53,"text":197},{"id":206,"depth":53,"text":207},"https:\u002F\u002Fjournal.aruodore.com\u002Fblog\u002Freproducible-gpu-benchmarks",null,"md","benchmark",[38],[228,229,230],"Separate device execution time from asynchronous host dispatch.","Report distributions and uncertainty rather than one attractive number.","Package enough environment data for a later comparison.",[232],"The examples focus on a single-device inference workload.",{},"2026-08-01","\u002Fblog\u002Freproducible-gpu-benchmarks",[237],"benchmark-protocol","2026-07-21",[240],{"type":241,"target":242},"develops","notebook:2026:gpu-synchronization-changed-the-result",{"title":5,"description":14},"reproducible-gpu-benchmarks","blog\u002Freproducible-gpu-benchmarks","GPU timing numbers become useful only when hardware, synchronization, sampling, and raw observations remain attached to the result.",[248,249,250],"gpu","reproducibility","performance",[252,253],"pytorch","cuda",[255,256],"scientific-computing","ai-engineering","1.1.0","GRfaGNg-Ub0yV-3qI3Q4zuB-BhqGmACoDIqe9nqJToo",1785815463879]