Add Llm.Runner project (#29)

Llm.sln

@@ -41,6 +41,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ci", "ci", "{655261D2-06EE-
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Llm.Benchmarks", "src\Llm.Benchmarks\Llm.Benchmarks.csproj", "{690F1276-E1B8-434C-AF45-B8C5BBB66B2A}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Llm.Runner", "src\Llm.Runner\Llm.Runner.csproj", "{5C43627A-0CC5-4222-B759-7469DAFB642C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -59,6 +61,10 @@ Global
 		{690F1276-E1B8-434C-AF45-B8C5BBB66B2A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{690F1276-E1B8-434C-AF45-B8C5BBB66B2A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{690F1276-E1B8-434C-AF45-B8C5BBB66B2A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5C43627A-0CC5-4222-B759-7469DAFB642C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5C43627A-0CC5-4222-B759-7469DAFB642C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5C43627A-0CC5-4222-B759-7469DAFB642C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5C43627A-0CC5-4222-B759-7469DAFB642C}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

benchmarks/AMD.Ryzen.7.PRO.7840U/VerifyTrain-Board.csv

@@ -0,0 +1,3 @@
+Name;Mean [ms]
+Llm_nietras;3540
+Llm;11928

benchmarks/AMD.Ryzen.7.PRO.7840U/VerifyTrain-Board.md

@@ -0,0 +1,4 @@
+|            Name|       Mean [ms]|
+|---------------:|---------------:|
+|     Llm_nietras|            3540|
+|             Llm|           11928|

benchmarks/AMD.Ryzen.9.5950X/VerifyTrain-Board.csv

@@ -0,0 +1,3 @@
+Name;Mean [ms]
+Llm_nietras;827
+Llm;4422

run.ps1

@@ -1 +1 @@
-dotnet run -c Release -f net8.0 --project src\Llm\Llm.csproj
+dotnet run -c Release -f net8.0 --project src\Llm.Runner\Llm.Runner.csproj

src/Llm.Runner/Llm.Runner.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <RootNamespace>nietras.LargeLanguageModel.Benchmarks</RootNamespace>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <DebugSymbols>true</DebugSymbols>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Llm\Llm.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
+    <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.14.0" />
+    <PackageReference Include="Sep" Version="0.5.3" />
+  </ItemGroup>
+
+</Project>

src/Llm.Runner/Program.cs

@@ -0,0 +1,142 @@
+// Type 'Program' can be sealed because it has no subtypes in its containing assembly and is not externally visible
+#pragma warning disable CA1852
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using BenchmarkDotNet.Environments;
+using nietras.LargeLanguageModel;
+using nietras.SeparatedValues;
+[assembly: System.Runtime.InteropServices.ComVisible(false)]
+
+Action<string> log = t => { Console.WriteLine(t); Trace.WriteLine(t); };
+
+var location = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+var dataDirectory = Path.Combine(location!, "../../../");
+
+log($"{Environment.Version} args: {args.Length}");
+
+var name = args?.Length > 0 ? args[0] : LlmFactory.DefaultName;
+log($"Llm '{name}'");
+var llm = LlmFactory.NameToCreate[name]();
+
+// Download the model and tokenizer files if they don't exist
+DownloadBinaryFilesIfNotExists(Gpt2.FileNames, Gpt2.RemoteUrl, dataDirectory, log);
+
+// Log to file too for reference
+var logFilePath = Path.Combine(dataDirectory, $"{name}.log");
+using var logWriter = new StreamWriter(logFilePath);
+Action<string> newLog = t => { log(t); logWriter.WriteLine(t); };
+
+const int steps = 10;
+var meanStep_ms = Gpt2.VerifyTrain(dataDirectory, llm, steps, newLog);
+var boardName = nameof(Gpt2.VerifyTrain);
+//Gpt2.Infer(dataDirectory, llm, newLog);
+//Gpt2.Train(dataDirectory, llm);
+
+var processorNameInDirectory = GetProcessorName();
+log(processorNameInDirectory);
+
+var sourceDirectory = GetSourceDirectory();
+var benchmarksDirectory = $"{sourceDirectory}/../../benchmarks/";
+var directory = $"{benchmarksDirectory}{processorNameInDirectory}";
+if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); }
+
+var filePathBoard = Path.Combine(directory, $"{boardName}-Board.csv");
+var filePathBoardMarkdown = Path.Combine(directory, $"{boardName}-Board.md");
+
+var (colNames, sortedBoard) = UpdateBoardCsv(name, meanStep_ms, filePathBoard);
+
+WriteBoardMarkdown(colNames, sortedBoard, filePathBoardMarkdown);
+
+static void WriteBoardMarkdown(string[] colNames, IReadOnlyList<string[]> sortedCols,
+    string filePathBoardMarkdown)
+{
+    using var writer = new StreamWriter(filePathBoardMarkdown);
+    writer.WriteLine($"|{string.Join("|", colNames.Select(c => c.PadLeft(16)))}|");
+    writer.WriteLine($"|{string.Join("|", colNames.Select(_ => ":").Select(c => c.PadLeft(16, '-')))}|");
+    foreach (var cols in sortedCols)
+    {
+        writer.WriteLine($"|{string.Join("|", cols.Select(c => c.PadLeft(16)))}|");
+    }
+}
+
+static void DownloadBinaryFilesIfNotExists(
+    IReadOnlyList<string> fileNames, Func<string, string> toUrl,
+    string dataDirectory, Action<string>? log)
+{
+    foreach (var fileName in fileNames)
+    {
+        var filePath = Path.Combine(dataDirectory, fileName);
+        filePath = Path.GetFullPath(filePath);
+        if (!File.Exists(filePath))
+        {
+            var url = toUrl(fileName);
+            log?.Invoke($"Downloading '{url}' to '{filePath}'");
+            using var client = new HttpClient();
+            // Download the file
+            var source = client.GetStreamAsync(url).Result;
+            using var destination = new FileStream(filePath, FileMode.Create);
+            source.CopyTo(destination);
+        }
+    }
+}
+
+static string GetSourceDirectory([CallerFilePath] string filePath = "") =>
+    Path.GetDirectoryName(filePath)!;
+
+static (string[] colNames, string[][] Cols) UpdateBoardCsv(
+    string name, double mean_ms, string filePathBoard)
+{
+    const string colNameName = "Name";
+    const string colNameMean = "Mean [ms]";
+
+    string[] colNames = [colNameName, colNameMean]; //, "StdDev [ms]", "Allocated [KB]"];
+
+    var value = (mean_ms, (string[])[name, mean_ms.ToString("F0")]);
+
+    var nameToCols = File.Exists(filePathBoard)
+        ? ReadNameToCols(filePathBoard, colNameName, colNameMean, colNames)
+        : [];
+    nameToCols[name] = value;
+
+    using var writerBoard = Sep.Writer().ToFile(filePathBoard);
+    var sorted = nameToCols.Values.OrderBy(v => v.Mean).ToArray();
+    foreach (var (_, cols) in sorted)
+    {
+        using var writeRow = writerBoard.NewRow();
+        writeRow[colNames].Set(cols);
+    }
+    return (colNames, sorted.Select(v => v.Cols).ToArray());
+}
+
+static Dictionary<string, (double Mean, string[] Cols)> ReadNameToCols(
+    string filePath, string colNameName, string colNameMean, string[] colNames)
+{
+    using var reader = Sep
+        .Reader(o => o with { Unescape = true, DisableFastFloat = true })
+        .FromFile(filePath);
+    return reader.Enumerate(r => (Name: r[colNameName].ToString(),
+            Mean: r[colNameMean].Parse<double>(), Cols: r[colNames].ToStringsArray()))
+        .ToDictionary(t => t.Name, t => (t.Mean, t.Cols));
+}
+
+static string GetProcessorName()
+{
+    var cpuInfo = HostEnvironmentInfo.GetCurrent().CpuInfo.Value;
+    var processorName = ProcessorBrandStringHelper.Prettify(cpuInfo);
+    var processorNameInDirectory = processorName
+        .Replace(" Processor", "").Replace(" CPU", "")
+        .Replace(" Graphics", "")
+        .Replace("/", "").Replace("\\", "")
+        .Replace(" ", ".");
+    // Remove iGPU info
+    var indexOfWith = processorNameInDirectory.LastIndexOf(".w.");
+    if (indexOfWith > 0)
+    { processorNameInDirectory = processorNameInDirectory.Substring(0, indexOfWith); }
+    return processorNameInDirectory;
+}

src/Llm/Gpt2.VerifyTrain.cs

@@ -30,7 +30,7 @@ public static ExpectedOutputTensors Create(int batchSize, int tokenCount, int vo
         public Tensor<float> ExpectedLogits { get; } = New([B, T, V], s);
     }
 
-    public static unsafe void VerifyTrain(string dataDirectory, ILlm llmToUse, int steps, Action<string>? log)
+    public static unsafe double VerifyTrain(string dataDirectory, ILlm llmToUse, int steps, Action<string>? log)
     {
         // build the GPT-2 model from a checkpoint
         using var model = ModelFromCheckpoint(dataDirectory + ModelBinaryFileName);
@@ -105,11 +105,13 @@ public static unsafe void VerifyTrain(string dataDirectory, ILlm llmToUse, int s
             }
             log?.Invoke($"All okay: {allOk}");
 
-            var timeReport = llm.CreateReport(steps - JitAndWarmupCount);
+            var (timeReport, meanStep_ms) = llm.CreateReport(steps - JitAndWarmupCount);
 
             log?.Invoke(timeReport);
 
             if (!allOk) { throw new ArithmeticException($"{llmToUse.GetType().Name} failed {nameof(Gpt2)} train test run, see output for details."); }
+
+            return meanStep_ms;
         }
     }
 

src/Llm/Llm.csproj

@@ -48,6 +48,9 @@
     <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
       <_Parameter1>$(MSBuildProjectName).Benchmarks</_Parameter1>
     </AssemblyAttribute>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>$(MSBuildProjectName).Runner</_Parameter1>
+    </AssemblyAttribute>
   </ItemGroup>
 
 </Project>

src/Llm/TimeLLm.cs

@@ -155,7 +155,7 @@ unsafe static void Clear<T>(T* ptr, nint size) where T : unmanaged
         NativeMemory.Clear(ptr, (nuint)(size * sizeof(T)));
     }
 
-    internal string CreateReport(int steps)
+    internal (string Report, double meanStep_ms) CreateReport(int steps)
     {
         var keyToStats = _keyToTimes.ToDictionary(p => p.Key, p => ComputeStats(p.Value));
         var totalSum_ms = keyToStats.Values.Sum(s => s.Sum_ms);
@@ -183,7 +183,7 @@ internal string CreateReport(int steps)
         {
             sb.AppendLine($"{method,-27} {sum_ms / totalSum_ms,4:P0} sum: {sum_ms,6:F0} [ms] per step: {sum_ms / steps,6:F0} [ms]");
         }
-        return sb.ToString();
+        return (sb.ToString(), totalSum_ms / steps);
     }
 
     static TimeStats ComputeStats(List<long> times)