update to inference engine

README.md

@@ -2,47 +2,29 @@
 license: mit
 library_name: unity-sentis
 pipeline_tag: text-generation
+tags:
+  - unity-inference-engine
 ---
 
+# Phi 1.5 in Unity 6 with Inference Engine
 
-# Phi 1.5 Model in Unity Sentis (Version 1.5.0-pre.2)
-*Version 1.3.0 Sentis files are not compatible with Sentis 1.5.0 and need to be recreated/downloaded
-
-This is the [Microsoft Phi 1.5](https://huggingface.co/microsoft/phi-1_5) model checked to run on Unity 2023. Phi 1.5 is a Large Language Model that was trained on synthesized data. Please see their page for more information about the model and license.
-The model has 1.3 billion parameters.
-
+This is the [Microsoft Phi 1.5](https://huggingface.co/microsoft/phi-1_5) running in Unity 6 with Inference Engine. Phi 1.5 is a Large Language Model trained on synthesized data. The model has 1.3 billion parameters.
 
 ## How to Use
-* Create a new scene in Unity 2023
-* Install `com.unity.sentis` version `1.5.0-pre.2` and `com.unity.nuget.newtonsoft-json` packages
-* Add the RunPhi15.cs file to the Main Camera
-* Put `phi15.sentis`, `vocab.json` and `merges.txt` in the Assets/StreamingAssets folder
-* Adjust some of the variables such as the `outputText` string to set the prompt
-* Press run
-* The output will appear in the console window (after some time. You may want to open the RAM inspector to see what is going on.)
-
-## Information
-This is the float32 version so it requires a lot of RAM (16GB) and VRAM (8GB). With less RAM it may take a long time to load. In the future we may add the float16 or uint8 quantized versions. This version also doesn't use caching of previous iterations so it is not the optimum speed but the implementation is simpler.
-
-## Example Input
-```
-Once upon a time, there were three
-```
-## Example Output
-```
-Once upon a time, there were three friends named Alice, Bob, and Carol. They were all passionate about mathematics and loved solving complex problems together. One day, they came across a challenging problem that required them to find the area of a triangle using the Pythagorean theorem.
-
-Alice, being the most experienced in geometry, took the lead and explained the steps to her friends. "To find the area of a triangle, we need to multiply the base by the height and divide the result by 2," she said confidently.
-
-Bob, who was always curious, asked, "But why do we divide the result by 2? Can't we just multiply the base and height?"
 
-Alice smiled and replied, "That's a great question, Bob. We divide by 2 because the area of a triangle is half of the product of its base and height. It's a fundamental concept in geometry."
+* Create a new scene in Unity 6;
+* Install `com.unity.ai.inference` from the package manager;
+* Install `com.unity.nuget.newtonsoft-json` from the package manager;
+* Add the `RunPhi15.cs` script to the Main Camera;
+* Drag the `phi15.sentis` asset from the `models` folder into the `Model Asset` field;
+* Drag the `vocab.json` asset from the `data` folder into the `Vocab Asset` field;
+* Drag the `merges.txt` asset from the `data` folder into the `Merges Asset` field;
 
-Carol, who had been listening intently, added, "So, if we have a triangle with...
-```
+## Preview
+Enter play mode. If working correctly the predicted text will be logged to the console.
 
-## Unity Sentis
-Unity Sentis is the inference engine which runs on Unity 2023. More can be found about it [here](https://unity.com/products/sentis)
+## Inference Engine
+Inference Engine is a neural network inference library for Unity. Find out more [here](https://docs.unity3d.com/Packages/com.unity.ai.inference@latest).
 
 ## Disclaimer
 Like any LLM, this model has the possibility to generate undesirable or untruthful text. Use at your discretion.

RunPhi15.cs

@@ -1,33 +1,15 @@
-using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
-using Unity.Sentis;
-using System.IO;
+using Unity.InferenceEngine;
 using System.Text;
-using FF = Unity.Sentis.Functional;
-
-/*
- *              Phi1.5 Inference Code
- *              ===========================
- *  
- *  Put this script on the Main Camera
- *  
- *  In Assets/StreamingAssets put:
- *  
- *  phi15.sentis (or put in asset folder)
- *  vocab.json
- *  merges.txt
- * 
- *  Install package com.unity.nuget.newtonsoft-json from packagemanger
- *  Install package com.unity.sentis
- * 
- */
-
-
-public class RunPhi15: MonoBehaviour
+using FF = Unity.InferenceEngine.Functional;
+
+public class RunPhi15 : MonoBehaviour
 {
-    //Drop the tinystories.sentis or onnx file on here if using an asset:
-    //public ModelAsset asset;
+    public ModelAsset modelAsset;
+    public TextAsset vocabAsset;
+    public TextAsset mergesAsset;
+
     const BackendType backend = BackendType.GPUCompute;
 
     //string outputString = "Once upon a time, there were three bears";
@@ -45,22 +27,21 @@ public class RunPhi15: MonoBehaviour
     //Store the vocabulary
     string[] tokens;
 
-    IWorker engine;
+    Worker engine;
 
-    int currentToken = 0;
+    int currentToken;
     int[] outputTokens = new int[maxTokens];
 
     // Used for special character decoding
     int[] whiteSpaceCharacters = new int[256];
     int[] encodedCharacters = new int[256];
 
-    bool runInference = false;
-
+    bool runInference;
 
     //stop after this many tokens
     const int stopAfter = 100;
 
-    int totalTokens = 0;
+    int totalTokens;
 
     string[] merges;
     Dictionary<string, int> vocab;
@@ -71,21 +52,17 @@ public class RunPhi15: MonoBehaviour
 
         LoadVocabulary();
 
-        var model1 = ModelLoader.Load(Path.Join(Application.streamingAssetsPath , "phi15.sentis"));
-
-        int outputIndex = model1.outputs.Count - 1;
-        //var model1 = ModelLoader.Load(asset);
+        var model1 = ModelLoader.Load(modelAsset);
         //Create a new model to select the random token:
-        var model2 = FF.Compile(
-            (input, currentToken) =>
-            {
-                var row = FF.Select(model1.Forward(input)[outputIndex], 1, currentToken);
-                return FF.Multinomial(predictability * row, 1);
-            },
-            (model1.inputs[0], InputDef.Int(new TensorShape()))
-        );
 
-        engine = WorkerFactory.CreateWorker(backend, model2);
+        var graph = new FunctionalGraph();
+        var input = graph.AddInput(model1, 0);
+        var currentTokenInput = graph.AddInput<int>(new TensorShape(), "currentToken");
+        var row = FF.Select(Functional.Forward(model1, input)[^1], 1, currentTokenInput);
+        var output = FF.Multinomial(predictability * row, 1);
+        var model2 = graph.Compile(output);
+
+        engine = new Worker(model2, backend);
 
         DecodePrompt(outputString);
 
@@ -103,15 +80,12 @@ public class RunPhi15: MonoBehaviour
 
     void RunInference()
     {
-        using var tokensSoFar = new TensorInt(new TensorShape(1, maxTokens), outputTokens);
-        using var index = new TensorInt(currentToken);
-
-        engine.Execute(new Dictionary<string, Tensor> { {"input_0", tokensSoFar },  { "input_1", index }});
+        using var tokensSoFar = new Tensor<int>(new TensorShape(1, maxTokens), outputTokens);
+        using var index = new Tensor<int>(new TensorShape(), new[] { currentToken });
 
-        var probs = engine.PeekOutput() as TensorInt;
-        //Debug.Log(probs.shape);
+        engine.Schedule(tokensSoFar, index);
 
-        probs.CompleteOperationsAndDownload();
+        using var probs = (engine.PeekOutput() as Tensor<int>).ReadbackAndClone();
 
         int ID = probs[0];
 
@@ -137,23 +111,22 @@ public class RunPhi15: MonoBehaviour
         else outputString += GetUnicodeText(tokens[ID]);
 
         Debug.Log(outputString);
-
     }
 
     void DecodePrompt(string text)
     {
         var inputTokens = GetTokens(text);
 
-        for(int i = 0; i < inputTokens.Count; i++)
+        for (int i = 0; i < inputTokens.Count; i++)
         {
             outputTokens[i] = inputTokens[i];
         }
         currentToken = inputTokens.Count - 1;
     }
-   
+
     void LoadVocabulary()
     {
-        var jsonText = File.ReadAllText(Path.Join(Application.streamingAssetsPath , "vocab.json"));
+        var jsonText = vocabAsset.text;
         vocab = Newtonsoft.Json.JsonConvert.DeserializeObject<Dictionary<string, int>>(jsonText);
         tokens = new string[vocab.Count];
         foreach (var item in vocab)
@@ -161,7 +134,7 @@ public class RunPhi15: MonoBehaviour
             tokens[item.Value] = item.Key;
         }
 
-        merges = File.ReadAllLines(Path.Join(Application.streamingAssetsPath , "merges.txt"));
+        merges = mergesAsset.text.Split("\r\n");
     }
 
     // Translates encoded special characters to Unicode
@@ -181,8 +154,7 @@ public class RunPhi15: MonoBehaviour
         string outText = "";
         foreach (char letter in text)
         {
-            outText += ((int)letter <= 256) ? letter :
-                (char)whiteSpaceCharacters[(int)(letter - 256)];
+            outText += (letter <= 256) ? letter : (char)whiteSpaceCharacters[letter - 256];
         }
         return outText;
     }
@@ -192,7 +164,7 @@ public class RunPhi15: MonoBehaviour
         string outText = "";
         foreach (char letter in text)
         {
-            outText += (char)encodedCharacters[(int)letter];
+            outText += (char)encodedCharacters[letter];
         }
         return outText;
     }
@@ -222,7 +194,7 @@ public class RunPhi15: MonoBehaviour
 
         // Start with a list of single characters
         var inputTokens = new List<string>();
-        foreach(var letter in text)
+        foreach (var letter in text)
         {
             inputTokens.Add(letter.ToString());
         }
@@ -231,7 +203,7 @@ public class RunPhi15: MonoBehaviour
 
         //Find the ids of the words in the vocab
         var ids = new List<int>();
-        foreach(var token in inputTokens)
+        foreach (var token in inputTokens)
         {
             if (vocab.TryGetValue(token, out int id))
             {
@@ -244,7 +216,7 @@ public class RunPhi15: MonoBehaviour
 
     void ApplyMerges(List<string> inputTokens)
     {
-        foreach(var merge in merges)
+        foreach (var merge in merges)
         {
             string[] pair = merge.Split(' ');
             int n = 0;
@@ -261,9 +233,8 @@ public class RunPhi15: MonoBehaviour
         }
     }
 
-    private void OnDestroy()
+    void OnDestroy()
     {
         engine?.Dispose();
     }
-    
 }

info.js

@@ -1,6 +0,0 @@
-{
-   "version" : [
-       "1.3.0-pre.3"
-   ]
-  
-}

info.json

@@ -0,0 +1,15 @@
+{
+    "code": [
+      "RunPhi15.cs"
+    ],
+    "models": [
+        "models/phi15.sentis"
+    ],
+    "data": [
+        "data/vocab.json",
+        "data/merges.txt"
+    ],
+    "version": [
+        "2.2.0"
+    ]
+}