From 3ff7e1ae2d46ca325ee7cbbc14388ec138f4273f Mon Sep 17 00:00:00 2001 From: Jean-Marc Valin Date: Sat, 3 Nov 2018 02:39:15 -0400 Subject: Adding the scripts used to train the RNN classifier Sorry, no doc for now --- training/rnn_dump.py | 66 +++++++++++++++++++ training/rnn_train.py | 177 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100755 training/rnn_dump.py create mode 100755 training/rnn_train.py (limited to 'training') diff --git a/training/rnn_dump.py b/training/rnn_dump.py new file mode 100755 index 00000000..c312088e --- /dev/null +++ b/training/rnn_dump.py @@ -0,0 +1,66 @@ +#!/usr/bin/python + +from __future__ import print_function + +from keras.models import Sequential +from keras.models import Model +from keras.layers import Input +from keras.layers import Dense +from keras.layers import LSTM +from keras.layers import GRU +from keras.models import load_model +from keras import backend as K +import sys + +import numpy as np + +def printVector(f, vector, name): + v = np.reshape(vector, (-1)); + #print('static const float ', name, '[', len(v), '] = \n', file=f) + f.write('static const opus_int8 {}[{}] = {{\n '.format(name, len(v))) + for i in range(0, len(v)): + f.write('{}'.format(max(-128,min(127,int(round(128*v[i])))))) + if (i!=len(v)-1): + f.write(',') + else: + break; + if (i%8==7): + f.write("\n ") + else: + f.write(" ") + #print(v, file=f) + f.write('\n};\n\n') + return; + +def binary_crossentrop2(y_true, y_pred): + return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_pred, y_true), axis=-1) + + +#model = load_model(sys.argv[1], custom_objects={'binary_crossentrop2': binary_crossentrop2}) +main_input = Input(shape=(None, 25), name='main_input') +x = Dense(32, activation='tanh')(main_input) +x = GRU(24, activation='tanh', recurrent_activation='sigmoid', return_sequences=True)(x) +x = Dense(2, activation='sigmoid')(x) +model = Model(inputs=main_input, outputs=x) +model.load_weights(sys.argv[1]) + +weights = model.get_weights() + +f = open(sys.argv[2], 'w') + +f.write('/*This file is automatically generated from a Keras model*/\n\n') +f.write('#ifdef HAVE_CONFIG_H\n#include "config.h"\n#endif\n\n#include "mlp.h"\n\n') + +printVector(f, weights[0], 'layer0_weights') +printVector(f, weights[1], 'layer0_bias') +printVector(f, weights[2], 'layer1_weights') +printVector(f, weights[3], 'layer1_recur_weights') +printVector(f, weights[4], 'layer1_bias') +printVector(f, weights[5], 'layer2_weights') +printVector(f, weights[6], 'layer2_bias') + +f.write('const DenseLayer layer0 = {\n layer0_bias,\n layer0_weights,\n 25, 32, 0\n};\n\n') +f.write('const GRULayer layer1 = {\n layer1_bias,\n layer1_weights,\n layer1_recur_weights,\n 32, 24\n};\n\n') +f.write('const DenseLayer layer2 = {\n layer2_bias,\n layer2_weights,\n 24, 2, 1\n};\n\n') + +f.close() diff --git a/training/rnn_train.py b/training/rnn_train.py new file mode 100755 index 00000000..29bcb034 --- /dev/null +++ b/training/rnn_train.py @@ -0,0 +1,177 @@ +#!/usr/bin/python3 + +from __future__ import print_function + +from keras.models import Sequential +from keras.models import Model +from keras.layers import Input +from keras.layers import Dense +from keras.layers import LSTM +from keras.layers import GRU +from keras.layers import CuDNNGRU +from keras.layers import SimpleRNN +from keras.layers import Dropout +from keras import losses +import h5py +from keras.optimizers import Adam + +from keras.constraints import Constraint +from keras import backend as K +import numpy as np + +import tensorflow as tf +from keras.backend.tensorflow_backend import set_session +config = tf.ConfigProto() +config.gpu_options.per_process_gpu_memory_fraction = 0.44 +set_session(tf.Session(config=config)) + +def binary_crossentrop2(y_true, y_pred): + return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_true, y_pred), axis=-1) + +def binary_accuracy2(y_true, y_pred): + return K.mean(K.cast(K.equal(y_true, K.round(y_pred)), 'float32') + K.cast(K.equal(y_true, 0.5), 'float32'), axis=-1) + +def quant_model(model): + weights = model.get_weights() + for k in range(len(weights)): + weights[k] = np.maximum(-128, np.minimum(127, np.round(128*weights[k])*0.0078125)) + model.set_weights(weights) + +class WeightClip(Constraint): + '''Clips the weights incident to each hidden unit to be inside a range + ''' + def __init__(self, c=2): + self.c = c + + def __call__(self, p): + return K.clip(p, -self.c, self.c) + + def get_config(self): + return {'name': self.__class__.__name__, + 'c': self.c} + +reg = 0.000001 +constraint = WeightClip(.998) + +print('Build model...') + +main_input = Input(shape=(None, 25), name='main_input') +x = Dense(32, activation='tanh', kernel_constraint=constraint, bias_constraint=constraint)(main_input) +#x = CuDNNGRU(24, return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x) +x = GRU(24, recurrent_activation='sigmoid', activation='tanh', return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x) +x = Dense(2, activation='sigmoid', kernel_constraint=constraint, bias_constraint=constraint)(x) +model = Model(inputs=main_input, outputs=x) + +batch_size = 2048 + +print('Loading data...') +with h5py.File('features10b.h5', 'r') as hf: + all_data = hf['data'][:] +print('done.') + +window_size = 1500 + +nb_sequences = len(all_data)//window_size +print(nb_sequences, ' sequences') +x_train = all_data[:nb_sequences*window_size, :-2] +x_train = np.reshape(x_train, (nb_sequences, window_size, 25)) + +y_train = np.copy(all_data[:nb_sequences*window_size, -2:]) +y_train = np.reshape(y_train, (nb_sequences, window_size, 2)) + +print("Marking ignores") +for s in y_train: + for e in s: + if (e[1] >= 1): + break + e[0] = 0.5 + +all_data = 0; +x_train = x_train.astype('float32') +y_train = y_train.astype('float32') + +print(len(x_train), 'train sequences. x shape =', x_train.shape, 'y shape = ', y_train.shape) + +model.load_weights('newweights10a1b_ep206.hdf5') + +#weights = model.get_weights() +#for k in range(len(weights)): +# weights[k] = np.round(128*weights[k])*0.0078125 +#model.set_weights(weights) + +# try using different optimizers and different optimizer configs +model.compile(loss=binary_crossentrop2, + optimizer=Adam(0.0001), + metrics=[binary_accuracy2]) + +print('Train...') +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=10, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep10.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=50, initial_epoch=10) +model.save("newweights10a1c_ep50.hdf5") + +model.compile(loss=binary_crossentrop2, + optimizer=Adam(0.0001), + metrics=[binary_accuracy2]) + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=100, initial_epoch=50) +model.save("newweights10a1c_ep100.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=150, initial_epoch=100) +model.save("newweights10a1c_ep150.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=200, initial_epoch=150) +model.save("newweights10a1c_ep200.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=201, initial_epoch=200) +model.save("newweights10a1c_ep201.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=202, initial_epoch=201, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep202.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=203, initial_epoch=202, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep203.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=204, initial_epoch=203, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep204.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=205, initial_epoch=204, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep205.hdf5") + +quant_model(model) +model.fit(x_train, y_train, + batch_size=batch_size, + epochs=206, initial_epoch=205, validation_data=(x_train, y_train)) +model.save("newweights10a1c_ep206.hdf5") + -- cgit v1.2.1