update to inference engine

README.md

@@ -2,26 +2,26 @@
 license: mit
 library_name: unity-sentis
 pipeline_tag: image-classification
+tags:
+  - unity-inference-engine
 ---
 
-## MobileNet V2 in Unity Sentis Format (Version 1.4.0-pre.2*)
-*Version 1.3.0 Sentis files are not compatible with 1.4.0 and need to be recreated/downloaded
 
-This is a small image classification model that works in Unity 2023. It is based on [MobileNet V2](https://arxiv.org/abs/1801.04381)
+# MobileNet V2 in Unity 6 with Inference Engine
+
+This is the [MobileNet V2](https://arxiv.org/abs/1801.04381) model running in Unity 6 with Inference Engine. This is a small image classification model.
 
 ## How to Use
-* Create a new scene in Unity 2023
-* Install `com.unity.sentis` version `1.4.0-pre.2` from the package manager
-* Add the C# script to the Main Camera
-* Drag the `mobilenet_v2.sentis` model onto the `modelAsset `field
-* Drag the `class_desc.txt` on to the `labelsAsset` field
-* Drag one of the sample images on to the inputImage field in the inspector.
-* Press play and the result of the prediction will print to the console window.
 
-# Example input
-![bee](Images/Bee.jpg)
-# Example output
-`Bee (100%)`
+* Create a new scene in Unity 6;
+* Install `com.unity.ai.inference` from the package manager;
+* Add the `RunMobileNet.cs` script to the Main Camera;
+* Drag the `mobilenet_v2.onnx` asset from the `models` folder into the `Model Asset` field;
+* Drag the `class_desc.txt` asset from the `data` folder into the `Labels Asset` field;
+* Drag an image, such as `Bee.jpg` asset from the `images` folder into the `Input Image` field;
+
+## Preview
+Enter play mode. If working correctly the predicted class will be logged to the console.
 
-## Unity Sentis
-Unity Sentis is the inference engine for Unity 3D. You can find more about it [here](https://unity.com/products/sentis)
+## Inference Engine
+Inference Engine is a neural network inference library for Unity. Find out more [here](https://docs.unity3d.com/Packages/com.unity.ai.inference@latest).

RunMobileNet.cs

@@ -1,71 +1,50 @@
-using System.Collections.Generic;
-using Unity.Sentis;
+using Unity.InferenceEngine;
 using UnityEngine;
-using System.IO;
-using FF = Unity.Sentis.Functional;
-/*
- *  MovileNetV2 Inference Script
- *  ============================
- *  
- *  Place this script on the Main Camera
- *  
- *  Drag an image to the inputImage field
- *  
- *  When run the prediction of what the image is will output to the console window.
- *  You can modify the script to make it do something more interesting.
- * 
- */
-
 
 public class RunMobileNet : MonoBehaviour
 {
-    //draw the sentis file here:
     public ModelAsset modelAsset;
 
-    const string modelName = "mobilenet_v2.sentis";
-
     //The image to classify here:
     public Texture2D inputImage;
 
     //Link class_desc.txt here:
     public TextAsset labelsAsset;
 
-    //All images are resized to these values to go into the model
-    const int imageHeight = 224;
-    const int imageWidth = 224;
+    //The input tensor
+    Tensor<float> input = new Tensor<float>(new TensorShape(1, 3, 224, 224));
 
     const BackendType backend = BackendType.GPUCompute;
 
-    private IWorker engine;
-    private string[] labels;
+    Worker worker;
+    string[] labels;
 
     //Used to normalise the input RGB values
-    TensorFloat mulRGB = new TensorFloat(new TensorShape(1, 3, 1, 1), new float[] { 1 / 0.229f, 1 / 0.224f, 1 / 0.225f });
-    TensorFloat shiftRGB = new TensorFloat(new TensorShape(1, 3, 1, 1), new float[] { 0.485f, 0.456f, 0.406f });
+    Tensor<float> mulRGB = new Tensor<float>(new TensorShape(1, 3, 1, 1), new[] { 1 / 0.229f, 1 / 0.224f, 1 / 0.225f });
+    Tensor<float> shiftRGB = new Tensor<float>(new TensorShape(1, 3, 1, 1), new[] { 0.485f, 0.456f, 0.406f });
 
     void Start()
     {
-
         //Parse neural net labels
         labels = labelsAsset.text.Split('\n');
 
-        //Load model from file or asset
-        //var model = ModelLoader.Load(Path.Join(Application.streamingAssetsPath, modelName));
+        //Load model from asset
         var model = ModelLoader.Load(modelAsset);
 
         //We modify the model to normalise the input RGB values and select the highest prediction
         //probability and item number
-        var model2 = FF.Compile(
-            input =>
-            {
-                var probability = model.Forward(NormaliseRGB(input))[0];
-                return (FF.ReduceMax(probability, 1), FF.ArgMax(probability, 1));
-            },
-            model.inputs[0]
-        );
-
-        //Setup the engine to run the model
-        engine = WorkerFactory.CreateWorker(backend, model2);
+        var graph = new FunctionalGraph();
+        var image = graph.AddInput(model, 0);
+        var normalizedInput = (image - Functional.Constant(shiftRGB)) * Functional.Constant(mulRGB);
+        var probability = Functional.Forward(model, normalizedInput)[0];
+        var value = Functional.ReduceMax(probability, 1);
+        var index = Functional.ArgMax(probability, 1);
+        graph.AddOutput(value, "value");
+        graph.AddOutput(index, "index");
+        var model2 = graph.Compile();
+
+        //Set up the worker to run the model
+        worker = new Worker(model2, backend);
 
         //Execute inference
         ExecuteML();
@@ -74,20 +53,18 @@ public class RunMobileNet : MonoBehaviour
     public void ExecuteML()
     {
         //Preprocess image for input
-        using var input = TextureConverter.ToTensor(inputImage, imageWidth, imageHeight, 3);
-        
-        //Execute neural net
-        engine.Execute(input);
+        TextureConverter.ToTensor(inputImage, input);
 
-        //Read output tensor
-        var probability = engine.PeekOutput("output_0") as TensorFloat;
-        var item = engine.PeekOutput("output_1") as TensorInt;
-        item.CompleteOperationsAndDownload();
-        probability.CompleteOperationsAndDownload();
+        //Schedule neural net
+        worker.Schedule(input);
+
+        //Read output tensors
+        using var value = (worker.PeekOutput("value") as Tensor<float>).ReadbackAndClone();
+        using var index = (worker.PeekOutput("index") as Tensor<int>).ReadbackAndClone();
 
         //Select the best output class and print the results
-        var ID = item[0];
-        var accuracy = probability[0];
+        var accuracy = value[0];
+        var ID = index[0];
 
         //The result is output to the console window
         int percent = Mathf.FloorToInt(accuracy * 100f + 0.5f);
@@ -97,16 +74,11 @@ public class RunMobileNet : MonoBehaviour
         Resources.UnloadUnusedAssets();
     }
 
-    //This scales and shifts the RGB values for input into the model
-    FunctionalTensor NormaliseRGB(FunctionalTensor image)
-    {
-        return (image - FunctionalTensor.FromTensor(shiftRGB)) * FunctionalTensor.FromTensor(mulRGB);
-    }
-    
-    private void OnDestroy()
+    void OnDestroy()
     {
+        input?.Dispose();
         mulRGB?.Dispose();
         shiftRGB?.Dispose();
-        engine?.Dispose();      
+        worker?.Dispose();
     }
 }

info.json

@@ -3,12 +3,12 @@
         "RunMobileNet.cs"
     ],
     "models": [
-        "mobilenet_v2.sentis"
+        "models/mobilenet_v2.onnx"
     ],
     "data":[
-        "class_desc.txt"
+        "data/class_desc.txt"
     ],
-    "version":[
-         "1.4.0"
+    "version": [
+        "2.2.0"
     ]
 }

mobilenet_v2.sentis

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:907d42cf325f7d2457b8ccdd12fabb5bea882d2757c3bb5bc57042e5ec6533bc
-size 13989036