NeuralPi

commit bf665b2952819d17dc3f48718068479587762e86
parent ce668f6d254b3f4cb9ccc2bdac59e7210e7dc60b
Author: keith <kbloemer89@gmail.com>
Date:   Thu, 21 Oct 2021 06:00:11 -0500

Added ability to load conditioned and snapshot models

Diffstat:
M
Source/PluginProcessor.cpp
 | 
24
+++++++++++++++++++++---
M
Source/RTNeuralLSTM.cpp
 | 
45
+++++++++++++++++++++++++++++++++++++++++++--
M
Source/RTNeuralLSTM.h
 | 
11
++++++++++-

3 files changed, 74 insertions(+), 6 deletions(-)
diff --git a/Source/PluginProcessor.cpp b/Source/PluginProcessor.cpp
@@ -214,8 +214,13 @@ void NeuralPiAudioProcessor::processBlock (AudioBuffer<float>& buffer, MidiBuffe
                 loadConfig(jsonFiles[model_index]);
                 current_model_index = model_index;
             }
-            //LSTM.process(buffer.getReadPointer(0), buffer.getWritePointer(0), numSamples);
-            LSTM.process(buffer.getReadPointer(0), gain, buffer.getWritePointer(0), numSamples);
+
+            // Process LSTM based on input_size (snapshot model or conditioned model)
+            if (LSTM.input_size == 1) {
+                LSTM.process(buffer.getReadPointer(0), buffer.getWritePointer(0), numSamples);
+            } else {
+                LSTM.process(buffer.getReadPointer(0), gain, buffer.getWritePointer(0), numSamples);
+            }
         }
 
         // Process IR
@@ -326,7 +331,20 @@ void NeuralPiAudioProcessor::loadConfig(File configFile)
     char_filename = path.toUTF8();
 
     try {
-        LSTM.load_json(char_filename);
+        // Check input size for conditioned models
+        // read JSON file
+        std::ifstream i2(char_filename);
+        nlohmann::json weights_json;
+        i2 >> weights_json;
+
+        int input_size_json = weights_json["/model_data/input_size"_json_pointer];
+        LSTM.input_size = input_size_json;
+        if (input_size_json == 1) {
+            LSTM.load_json(char_filename);
+        }
+        else {
+            LSTM.load_json2(char_filename);
+        }
         model_loaded = 1;
     }
     catch (const std::exception& e) {
diff --git a/Source/RTNeuralLSTM.cpp b/Source/RTNeuralLSTM.cpp
@@ -19,6 +19,7 @@ Vec2d transpose(const Vec2d& x)
 
 void RT_LSTM::load_json(const char* filename)
 {
+
     auto& lstm = model.get<0>();
     auto& dense = model.get<1>();
 
@@ -46,9 +47,49 @@ void RT_LSTM::load_json(const char* filename)
     dense.setBias(dense_bias.data());
 }
 
+void RT_LSTM::load_json2(const char* filename)
+{
+
+    auto& lstm = model_cond1.get<0>();
+    auto& dense = model_cond1.get<1>();
+
+    // read a JSON file
+    std::ifstream i2(filename);
+    nlohmann::json weights_json;
+    i2 >> weights_json;
+
+    Vec2d lstm_weights_ih = weights_json["/state_dict/rec.weight_ih_l0"_json_pointer];
+    lstm.setWVals(transpose(lstm_weights_ih));
+
+    Vec2d lstm_weights_hh = weights_json["/state_dict/rec.weight_hh_l0"_json_pointer];
+    lstm.setUVals(transpose(lstm_weights_hh));
+
+    std::vector<float> lstm_bias_ih = weights_json["/state_dict/rec.bias_ih_l0"_json_pointer];
+    std::vector<float> lstm_bias_hh = weights_json["/state_dict/rec.bias_hh_l0"_json_pointer];
+    for (int i = 0; i < 80; ++i)
+        lstm_bias_hh[i] += lstm_bias_ih[i];
+    lstm.setBVals(lstm_bias_hh);
+
+    Vec2d dense_weights = weights_json["/state_dict/lin.weight"_json_pointer];
+    dense.setWeights(dense_weights);
+
+    std::vector<float> dense_bias = weights_json["/state_dict/lin.bias"_json_pointer];
+    dense.setBias(dense_bias.data());
+}
+
 void RT_LSTM::reset()
 {
-    model.reset();
+    if (input_size == 1) {
+        model.reset();
+    } else {
+        model_cond1.reset();
+    }
+}
+
+void RT_LSTM::process(const float* inData, float* outData, int numSamples)
+{
+    for (int i = 0; i < numSamples; ++i)
+        outData[i] = model.forward(inData + i) + inData[i];
 }
 
 void RT_LSTM::process(const float* inData, float param, float* outData, int numSamples)
@@ -56,7 +97,7 @@ void RT_LSTM::process(const float* inData, float param, float* outData, int numS
     for (int i = 0; i < numSamples; ++i) {
         inArray[0] = inData[i];
         inArray[1] = param;
-        outData[i] = model.forward(inArray) + inData[i];
+        outData[i] = model_cond1.forward(inArray) + inData[i];
     }
 
 }
diff --git a/Source/RTNeuralLSTM.h b/Source/RTNeuralLSTM.h
@@ -9,12 +9,21 @@ public:
 
     void reset();
     void load_json(const char* filename);
+    void load_json2(const char* filename);
 
+    void process(const float* inData, float* outData, int numSamples);
     void process(const float* inData, float param, float* outData, int numSamples);
+
+    int input_size = 1;
     
 private:
+    RTNeural::ModelT<float, 1, 1,
+        RTNeural::LSTMLayerT<float, 1, 20>,
+        RTNeural::DenseT<float, 20, 1>> model;
+
     RTNeural::ModelT<float, 2, 1,
         RTNeural::LSTMLayerT<float, 2, 20>,
-        RTNeural::DenseT<float, 20, 1>> model;
+        RTNeural::DenseT<float, 20, 1>> model_cond1;
+
     float inArray[2] = { 0.0, 0.0 };
 };