# -*- coding: utf-8 -*-
from logging import getLogger
import numpy as np
from pysummarization.abstractable_semantics import AbstractableSemantics
from pysummarization.vectorizable_token import VectorizableToken
# LSTM Graph which is-a `Synapse`.
from pydbm.synapse.recurrenttemporalgraph.lstm_graph import LSTMGraph as EncoderGraph
from pydbm.synapse.recurrenttemporalgraph.lstm_graph import LSTMGraph as DecoderGraph
from pydbm.synapse.recurrenttemporalgraph.lstm_graph import LSTMGraph as ReEncoderGraph
# Loss function.
from pydbm.loss.mean_squared_error import MeanSquaredError
# Adam as a Loss function.
from pydbm.optimization.optparams.nadam import Nadam as EncoderAdam
from pydbm.optimization.optparams.nadam import Nadam as DecoderAdam
from pydbm.optimization.optparams.nadam import Nadam as ReEncoderAdam
# Verification.
from pydbm.verification.verificate_function_approximation import VerificateFunctionApproximation
# LSTM model.
from pydbm.rnn.lstm_model import LSTMModel
from pydbm.rnn.lstm_model import LSTMModel as Encoder
from pydbm.rnn.lstm_model import LSTMModel as Decoder
from pydbm.rnn.lstm_model import LSTMModel as ReEncoder
# Logistic Function as activation function.
from pydbm.activation.logistic_function import LogisticFunction
# Tanh Function as activation function.
from pydbm.activation.tanh_function import TanhFunction
# Softmax Function as activation function.
from pydbm.activation.softmax_function import SoftmaxFunction
# Encoder/Decoder
from pydbm.rnn.encoder_decoder_controller import EncoderDecoderController
# Iterator/Generator.
from pydbm.cnn.feature_generator import FeatureGenerator
[docs]class ReSeq2Seq(AbstractableSemantics):
'''
A retrospective sequence-to-sequence learning(re-seq2seq).
The concept of the re-seq2seq(Zhang, K. et al., 2018) provided inspiration to this library.
This model is a new sequence learning model mainly in the field of Video Summarizations.
"The key idea behind re-seq2seq is to measure how well the machine-generated summary
is similar to the original video in an abstract semantic space" (Zhang, K. et al., 2018, p3).
The encoder of a seq2seq model observes the original video and output feature points
which represents the semantic meaning of the observed data points.
Then the feature points is observed by the decoder of this model.
Additionally, in the re-seq2seq model, the outputs of the decoder is propagated
to a retrospective encoder, which infers feature points to represent the
semantic meaning of the summary. "If the summary preserves the important and
relevant information in the original video, then we should expect that the
two embeddings are similar (e.g. in Euclidean distance)" (Zhang, K. et al., 2018, p3).
This library refers to this intuitive insight above to apply the model to text summarizations.
Like videos, semantic feature representation based on representation learning of manifolds
is also possible in text summarizations.
The intuition in the design of their loss function is also suggestive.
"The intuition behind our modeling is that the outputs should convey
the same amount of information as the inputs. For summarization,
this is precisely the goal: a good summary should be such that after viewing
the summary, users would get about the same amount of information as if they
had viewed the original video" (Zhang, K. et al., 2018, p7).
But the model in this library and Zhang, K. et al.(2018) are different in some respects
from the relation with the specification of the Deep Learning library: [pydbm](https://github.com/chimera0/accel-brain-code/tree/master/Deep-Learning-by-means-of-Design-Pattern).
First, Encoder/Decoder based on LSTM is not designed as a hierarchical structure.
Second, it is possible to introduce regularization techniques which are not discussed in
Zhang, K. et al.(2018) such as the dropout, the gradient clipping, and limitation of weights.
Third, the regression loss function for matching summaries is simplified in terms of
calculation efficiency in this library.
References:
- Zhang, K., Grauman, K., & Sha, F. (2018). Retrospective Encoders for Video Summarization. In Proceedings of the European Conference on Computer Vision (ECCV) (pp. 383-399).
'''
# Logs of accuracy.
__logs_tuple_list = []
def __init__(
self,
margin_param=0.01,
retrospective_lambda=0.5,
retrospective_eta=0.5,
encoder_decoder_controller=None,
retrospective_encoder=None,
input_neuron_count=20,
hidden_neuron_count=20,
weight_limit=1e+10,
dropout_rate=0.5,
pre_learning_epochs=1000,
epochs=100,
batch_size=20,
learning_rate=1e-05,
learning_attenuate_rate=1.0,
attenuate_epoch=50,
grad_clip_threshold=1e+10,
seq_len=8,
bptt_tau=8,
test_size_rate=0.3,
tol=0.0,
tld=100.0
):
'''
Init.
Args:
margin_param: A margin parameter for the mismatched pairs penalty.
retrospective_lambda: Tradeoff parameter for loss function.
retrospective_eta: Tradeoff parameter for loss function.
encoder_decoder_controller: is-a `EncoderDecoderController`.
retrospective_encoder: is-a `LSTMModel` as a retrospective encoder(or re-encoder).
input_neuron_count: The number of units in input layers.
hidden_neuron_count: The number of units in hidden layers.
weight_limit: Regularization for weights matrix to repeat multiplying
the weights matrix and `0.9` until $\sum_{j=0}^{n}w_{ji}^2 < weight\_limit$.
dropout_rate: Probability of dropout.
pre_learning_epochs: The epochs in mini-batch pre-learning Encoder/Decoder.
If this value is `0`, no pre-learning will be executed
in this class's method `learn`. In this case, you should
do pre-learning before calling `learn`.
epochs: The epochs in mini-batch training Encoder/Decoder and retrospective encoder.
batch_size: Batch size.
learning_rate: Learning rate.
learning_attenuate_rate: Attenuate the `learning_rate` by a factor of this value every `attenuate_epoch`.
attenuate_epoch: Attenuate the `learning_rate` by a factor of `learning_attenuate_rate` every `attenuate_epoch`.
Additionally, in relation to regularization,
this class constrains weight matrixes every `attenuate_epoch`.
grad_clip_threshold: Threshold of the gradient clipping.
seq_len: The length of sequneces in Decoder with Attention model.
bptt_tau: Refereed maxinum step `t` in Backpropagation Through Time(BPTT).
If `0`, this class referes all past data in BPTT.
test_size_rate: Size of Test data set. If this value is `0`, the validation will not be executed.
tol: Tolerance for the optimization.
When the loss or score is not improving by at least tol
for two consecutive iterations, convergence is considered
to be reached and training stops.
tld: Tolerance for deviation of loss.
'''
if isinstance(margin_param, float) is False:
raise TypeError("The type of `margin_param` must be `float`.")
if margin_param <= 0:
raise ValueError("The value of `margin_param` must be more than `0`.")
self.__margin_param = margin_param
if isinstance(retrospective_lambda, float) is False or isinstance(retrospective_eta, float) is False:
raise TypeError("The type of `retrospective_lambda` and `retrospective_eta` must be `float`.")
if retrospective_lambda < 0 or retrospective_eta < 0:
raise ValueError("The values of `retrospective_lambda` and `retrospective_eta` must be more then `0`.")
if retrospective_lambda + retrospective_eta != 1:
raise ValueError("The sum of `retrospective_lambda` and `retrospective_eta` must be `1`.")
self.__retrospective_lambda = retrospective_lambda
self.__retrospective_eta = retrospective_eta
if encoder_decoder_controller is None:
encoder_decoder_controller = self.__build_encoder_decoder_controller(
input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
weight_limit=weight_limit,
dropout_rate=dropout_rate,
epochs=pre_learning_epochs,
batch_size=batch_size,
learning_rate=learning_rate,
attenuate_epoch=attenuate_epoch,
learning_attenuate_rate=learning_attenuate_rate,
seq_len=seq_len,
bptt_tau=bptt_tau,
test_size_rate=test_size_rate,
tol=tol,
tld=tld
)
else:
if isinstance(encoder_decoder_controller, EncoderDecoderController) is False:
raise TypeError()
if retrospective_encoder is None:
retrospective_encoder = self.__build_retrospective_encoder(
input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
weight_limit=weight_limit,
dropout_rate=dropout_rate,
batch_size=batch_size,
learning_rate=learning_rate,
bptt_tau=bptt_tau
)
else:
if isinstance(retrospective_encoder, LSTMModel) is False:
raise TypeError()
self.__encoder_decoder_controller = encoder_decoder_controller
self.__retrospective_encoder = retrospective_encoder
self.__epochs = epochs
self.__batch_size = batch_size
self.__learning_rate = learning_rate
self.__attenuate_epoch = attenuate_epoch
self.__learning_attenuate_rate = learning_attenuate_rate
self.__grad_clip_threshold = grad_clip_threshold
self.__test_size_rate = test_size_rate
self.__pre_learning_epochs = pre_learning_epochs
self.__tol = tol
self.__tld = tld
self.__input_neuron_count = input_neuron_count
self.__hidden_neuron_count = hidden_neuron_count
logger = getLogger("pysummarization")
self.__logger = logger
self.__logs_tuple_list = []
def __build_encoder_decoder_controller(
self,
input_neuron_count=20,
hidden_neuron_count=20,
weight_limit=1e+15,
dropout_rate=0.5,
epochs=1000,
batch_size=20,
learning_rate=1e-05,
attenuate_epoch=50,
learning_attenuate_rate=1.0,
seq_len=8,
bptt_tau=8,
test_size_rate=0.3,
tol=1e-10,
tld=100.0
):
encoder_graph = EncoderGraph()
encoder_graph.observed_activating_function = LogisticFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
encoder_graph.create_rnn_cells(
input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1
)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = Encoder(
graph=encoder_graph,
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=1.0,
attenuate_epoch=50,
bptt_tau=8,
test_size_rate=0.3,
computable_loss=MeanSquaredError(),
opt_params=encoder_opt_params,
verificatable_result=VerificateFunctionApproximation(),
tol=tol,
tld=tld
)
decoder_graph = DecoderGraph()
decoder_graph.observed_activating_function = LogisticFunction()
decoder_graph.input_gate_activating_function = LogisticFunction()
decoder_graph.forget_gate_activating_function = LogisticFunction()
decoder_graph.output_gate_activating_function = LogisticFunction()
decoder_graph.hidden_activating_function = LogisticFunction()
decoder_graph.output_activating_function = SoftmaxFunction()
decoder_graph.create_rnn_cells(
input_neuron_count=hidden_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=input_neuron_count
)
decoder_opt_params = DecoderAdam()
decoder_opt_params.weight_limit = weight_limit
decoder_opt_params.dropout_rate = dropout_rate
decoder = Decoder(
graph=decoder_graph,
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=1.0,
attenuate_epoch=50,
seq_len=seq_len,
bptt_tau=bptt_tau,
test_size_rate=0.3,
computable_loss=MeanSquaredError(),
opt_params=decoder_opt_params,
verificatable_result=VerificateFunctionApproximation()
)
encoder_decoder_controller = EncoderDecoderController(
encoder=encoder,
decoder=decoder,
epochs=epochs,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=learning_attenuate_rate,
attenuate_epoch=attenuate_epoch,
test_size_rate=test_size_rate,
computable_loss=MeanSquaredError(),
verificatable_result=VerificateFunctionApproximation(),
tol=tol,
tld=tld
)
return encoder_decoder_controller
def __build_retrospective_encoder(
self,
input_neuron_count=20,
hidden_neuron_count=20,
weight_limit=1e+10,
dropout_rate=0.5,
batch_size=20,
learning_rate=1e-05,
bptt_tau=8
):
encoder_graph = ReEncoderGraph()
encoder_graph.observed_activating_function = TanhFunction()
encoder_graph.input_gate_activating_function = LogisticFunction()
encoder_graph.forget_gate_activating_function = LogisticFunction()
encoder_graph.output_gate_activating_function = LogisticFunction()
encoder_graph.hidden_activating_function = LogisticFunction()
encoder_graph.output_activating_function = LogisticFunction()
encoder_graph.create_rnn_cells(
input_neuron_count=input_neuron_count,
hidden_neuron_count=hidden_neuron_count,
output_neuron_count=1
)
encoder_opt_params = EncoderAdam()
encoder_opt_params.weight_limit = weight_limit
encoder_opt_params.dropout_rate = dropout_rate
encoder = ReEncoder(
graph=encoder_graph,
epochs=100,
batch_size=batch_size,
learning_rate=learning_rate,
learning_attenuate_rate=1.0,
attenuate_epoch=50,
bptt_tau=bptt_tau,
test_size_rate=0.3,
computable_loss=MeanSquaredError(),
opt_params=encoder_opt_params,
verificatable_result=VerificateFunctionApproximation()
)
return encoder
[docs] def learn(self, observed_arr, target_arr):
'''
Training the model.
Args:
observed_arr: `np.ndarray` of observed data points.
target_arr: `np.ndarray` of target labeled data.
'''
# Pre-learning.
if self.__pre_learning_epochs > 0:
self.__encoder_decoder_controller.learn(observed_arr, observed_arr)
learning_rate = self.__learning_rate
row_o = observed_arr.shape[0]
row_t = target_arr.shape[0]
if row_t != 0 and row_t != row_o:
raise ValueError("The row of `target_arr` must be equivalent to the row of `observed_arr`.")
if row_t == 0:
target_arr = observed_arr.copy()
else:
if target_arr.ndim == 2:
target_arr = target_arr.reshape((target_arr.shape[0], 1, target_arr.shape[1]))
if self.__test_size_rate > 0:
train_index = np.random.choice(observed_arr.shape[0], round((1 - self.__test_size_rate) * observed_arr.shape[0]), replace=False)
test_index = np.array(list(set(range(observed_arr.shape[0])) - set(train_index)))
train_observed_arr = observed_arr[train_index]
test_observed_arr = observed_arr[test_index]
train_target_arr = target_arr[train_index]
test_target_arr = target_arr[test_index]
else:
train_observed_arr = observed_arr
train_target_arr = target_arr
encoder_best_params_list = []
decoder_best_params_list = []
re_encoder_best_params_list = []
try:
self.__change_inferencing_mode(False)
self.__memory_tuple_list = []
eary_stop_flag = False
loss_list = []
min_loss = None
for epoch in range(self.__epochs):
if ((epoch + 1) % self.__attenuate_epoch == 0):
learning_rate = learning_rate * self.__learning_attenuate_rate
rand_index = np.random.choice(train_observed_arr.shape[0], size=self.__batch_size)
batch_observed_arr = train_observed_arr[rand_index]
batch_target_arr = train_target_arr[rand_index]
try:
_ = self.inference(batch_observed_arr)
delta_arr, _, loss = self.compute_retrospective_loss()
remember_flag = False
if len(loss_list) > 0:
if abs(loss - (sum(loss_list)/len(loss_list))) > self.__tld:
remember_flag = True
if remember_flag is True:
self.__remember_best_params(
encoder_best_params_list,
decoder_best_params_list,
re_encoder_best_params_list
)
# Re-try.
_ = self.inference(batch_observed_arr)
delta_arr, _, loss = self.compute_retrospective_loss()
re_encoder_grads_list, decoder_grads_list, encoder_delta_arr, encoder_grads_list = self.back_propagation(delta_arr)
self.optimize(
re_encoder_grads_list,
decoder_grads_list,
encoder_grads_list,
learning_rate,
epoch
)
if min_loss is None or min_loss > loss:
min_loss = loss
encoder_best_params_list = [
self.__encoder_decoder_controller.encoder.graph.weights_lstm_hidden_arr,
self.__encoder_decoder_controller.encoder.graph.weights_lstm_observed_arr,
self.__encoder_decoder_controller.encoder.graph.lstm_bias_arr
]
decoder_best_params_list = [
self.__encoder_decoder_controller.decoder.graph.weights_lstm_hidden_arr,
self.__encoder_decoder_controller.decoder.graph.weights_lstm_observed_arr,
self.__encoder_decoder_controller.decoder.graph.lstm_bias_arr
]
re_encoder_best_params_list = [
self.__retrospective_encoder.graph.weights_lstm_hidden_arr,
self.__retrospective_encoder.graph.weights_lstm_observed_arr,
self.__retrospective_encoder.graph.lstm_bias_arr
]
self.__logger.debug("Best params are updated.")
self.__encoder_decoder_controller.encoder.graph.hidden_activity_arr = np.array([])
self.__encoder_decoder_controller.encoder.graph.cec_activity_arr = np.array([])
self.__encoder_decoder_controller.decoder.graph.hidden_activity_arr = np.array([])
self.__encoder_decoder_controller.decoder.graph.cec_activity_arr = np.array([])
self.__retrospective_encoder.graph.hidden_activity_arr = np.array([])
self.__retrospective_encoder.graph.cec_activity_arr = np.array([])
except FloatingPointError:
if epoch > int(self.__epochs * 0.7):
self.__logger.debug(
"Underflow occurred when the parameters are being updated. Because of early stopping, this error is catched and the parameter is not updated."
)
eary_stop_flag = True
break
else:
self.__logger.debug(
"Underflow occurred when the parameters are being updated."
)
raise
if self.__test_size_rate > 0:
rand_index = np.random.choice(test_observed_arr.shape[0], size=self.__batch_size)
test_batch_observed_arr = test_observed_arr[rand_index]
test_batch_target_arr = test_target_arr[rand_index]
self.__change_inferencing_mode(True)
_ = self.inference(test_batch_observed_arr)
_, _, test_loss = self.compute_retrospective_loss()
remember_flag = False
if len(loss_list) > 0:
if abs(test_loss - (sum(loss_list)/len(loss_list))) > self.__tld:
remember_flag = True
if remember_flag is True:
self.__remember_best_params(
encoder_best_params_list,
decoder_best_params_list,
re_encoder_best_params_list
)
# Re-try.
_ = self.inference(test_batch_observed_arr)
_, _, test_loss = self.compute_retrospective_loss()
self.__change_inferencing_mode(False)
self.__verificate_retrospective_loss(loss, test_loss)
self.__encoder_decoder_controller.encoder.graph.hidden_activity_arr = np.array([])
self.__encoder_decoder_controller.encoder.graph.cec_activity_arr = np.array([])
self.__encoder_decoder_controller.decoder.graph.hidden_activity_arr = np.array([])
self.__encoder_decoder_controller.decoder.graph.cec_activity_arr = np.array([])
if epoch > 1 and abs(loss - loss_list[-1]) < self.__tol:
eary_stop_flag = True
break
loss_list.append(loss)
except KeyboardInterrupt:
self.__logger.debug("Interrupt.")
if eary_stop_flag is True:
self.__logger.debug("Early stopping.")
eary_stop_flag = False
self.__remember_best_params(
encoder_best_params_list,
decoder_best_params_list,
re_encoder_best_params_list
)
self.__change_inferencing_mode(True)
self.__logger.debug("end. ")
[docs] def learn_generated(self, feature_generator):
'''
Learn features generated by `FeatureGenerator`.
Args:
feature_generator: is-a `FeatureGenerator`.
'''
if isinstance(feature_generator, FeatureGenerator) is False:
raise TypeError("The type of `feature_generator` must be `FeatureGenerator`.")
# Pre-learning.
if self.__pre_learning_epochs > 0:
self.__encoder_decoder_controller.learn_generated(feature_generator)
learning_rate = self.__learning_rate
encoder_best_params_list = []
decoder_best_params_list = []
re_encoder_best_params_list = []
try:
self.__change_inferencing_mode(False)
self.__memory_tuple_list = []
eary_stop_flag = False
loss_list = []
min_loss = None
epoch = 0
for batch_observed_arr, batch_target_arr, test_batch_observed_arr, test_batch_target_arr in feature_generator.generate():
epoch += 1
if ((epoch + 1) % self.__attenuate_epoch == 0):
learning_rate = learning_rate * self.__learning_attenuate_rate
try:
_ = self.inference(batch_observed_arr)
delta_arr, _, loss = self.compute_retrospective_loss()
remember_flag = False
if len(loss_list) > 0:
if abs(loss - (sum(loss_list)/len(loss_list))) > self.__tld:
remember_flag = True
if remember_flag is True:
self.__remember_best_params(
encoder_best_params_list,
decoder_best_params_list,
re_encoder_best_params_list
)
# Re-try.
_ = self.inference(batch_observed_arr)
delta_arr, _, loss = self.compute_retrospective_loss()
re_encoder_grads_list, decoder_grads_list, encoder_delta_arr, encoder_grads_list = self.back_propagation(delta_arr)
self.optimize(
re_encoder_grads_list,
decoder_grads_list,
encoder_grads_list,
learning_rate,
epoch
)
if min_loss is None or min_loss > loss:
min_loss = loss
encoder_best_params_list = [
self.__encoder_decoder_controller.encoder.graph.weights_lstm_hidden_arr,
self.__encoder_decoder_controller.encoder.graph.weights_lstm_observed_arr,
self.__encoder_decoder_controller.encoder.graph.lstm_bias_arr
]
decoder_best_params_list = [
self.__encoder_decoder_controller.decoder.graph.weights_lstm_hidden_arr,
self.__encoder_decoder_controller.decoder.graph.weights_lstm_observed_arr,
self.__encoder_decoder_controller.decoder.graph.lstm_bias_arr
]
re_encoder_best_params_list = [
self.__retrospective_encoder.graph.weights_lstm_hidden_arr,
self.__retrospective_encoder.graph.weights_lstm_observed_arr,
self.__retrospective_encoder.graph.lstm_bias_arr
]
self.__logger.debug("Best params are updated.")
self.__encoder_decoder_controller.encoder.graph.hidden_activity_arr = np.array([])
self.__encoder_decoder_controller.encoder.graph.cec_activity_arr = np.array([])
self.__encoder_decoder_controller.decoder.graph.hidden_activity_arr = np.array([])
self.__encoder_decoder_controller.decoder.graph.cec_activity_arr = np.array([])
self.__retrospective_encoder.graph.hidden_activity_arr = np.array([])
self.__retrospective_encoder.graph.cec_activity_arr = np.array([])
except FloatingPointError:
if epoch > int(self.__epochs * 0.7):
self.__logger.debug(
"Underflow occurred when the parameters are being updated. Because of early stopping, this error is catched and the parameter is not updated."
)
eary_stop_flag = True
break
else:
self.__logger.debug(
"Underflow occurred when the parameters are being updated."
)
raise
if self.__test_size_rate > 0:
self.__change_inferencing_mode(True)
_ = self.inference(test_batch_observed_arr)
_, _, test_loss = self.compute_retrospective_loss()
remember_flag = False
if len(loss_list) > 0:
if abs(test_loss - (sum(loss_list)/len(loss_list))) > self.__tld:
remember_flag = True
if remember_flag is True:
self.__remember_best_params(
encoder_best_params_list,
decoder_best_params_list,
re_encoder_best_params_list
)
# Re-try.
_ = self.inference(test_batch_observed_arr)
_, _, test_loss = self.compute_retrospective_loss()
self.__change_inferencing_mode(False)
self.__verificate_retrospective_loss(loss, test_loss)
self.__encoder_decoder_controller.encoder.graph.hidden_activity_arr = np.array([])
self.__encoder_decoder_controller.encoder.graph.cec_activity_arr = np.array([])
self.__encoder_decoder_controller.decoder.graph.hidden_activity_arr = np.array([])
self.__encoder_decoder_controller.decoder.graph.cec_activity_arr = np.array([])
if epoch > 1 and abs(loss - loss_list[-1]) < self.__tol:
eary_stop_flag = True
break
loss_list.append(loss)
except KeyboardInterrupt:
self.__logger.debug("Interrupt.")
if eary_stop_flag is True:
self.__logger.debug("Early stopping.")
eary_stop_flag = False
self.__remember_best_params(
encoder_best_params_list,
decoder_best_params_list,
re_encoder_best_params_list
)
self.__change_inferencing_mode(True)
self.__logger.debug("end. ")
[docs] def inference(self, observed_arr):
'''
Infernece by the model.
Args:
observed_arr: `np.ndarray` of observed data points.
Returns:
`np.ndarray` of inferenced feature points.
'''
decoded_arr = self.__encoder_decoder_controller.inference(observed_arr)
encoded_arr = self.__encoder_decoder_controller.get_feature_points()
_ = self.__retrospective_encoder.inference(decoded_arr)
re_encoded_arr = self.__retrospective_encoder.get_feature_points()
self.__inferenced_tuple = (observed_arr, encoded_arr, decoded_arr, re_encoded_arr)
return re_encoded_arr
[docs] def summarize(self, test_arr, vectorizable_token, sentence_list, limit=5):
'''
Summarize input document.
Args:
test_arr: `np.ndarray` of observed data points..
vectorizable_token: is-a `VectorizableToken`.
sentence_list: `list` of all sentences.
limit: The number of selected abstract sentence.
Returns:
`list` of `str` of abstract sentences.
'''
if isinstance(vectorizable_token, VectorizableToken) is False:
raise TypeError()
_ = self.inference(test_arr)
_, loss_arr, _ = self.compute_retrospective_loss()
loss_list = loss_arr.tolist()
abstract_list = []
for i in range(limit):
key = loss_arr.argmin()
_ = loss_list.pop(key)
loss_arr = np.array(loss_list)
seq_arr = test_arr[key]
token_arr = vectorizable_token.tokenize(seq_arr.tolist())
s = " ".join(token_arr.tolist())
_s = "".join(token_arr.tolist())
for sentence in sentence_list:
if s in sentence or _s in sentence:
abstract_list.append(sentence)
abstract_list = list(set(abstract_list))
if len(abstract_list) >= limit:
break
return abstract_list
[docs] def back_propagation(self, delta_arr):
'''
Back propagation.
Args:
delta_output_arr: Delta.
Returns:
Tuple data.
- decoder's `list` of gradations,
- encoder's `np.ndarray` of Delta,
- encoder's `list` of gradations.
'''
re_encoder_delta_arr, delta_hidden_arr, re_encoder_grads_list = self.__retrospective_encoder.hidden_back_propagate(
delta_arr[:, -1]
)
re_encoder_grads_list.insert(0, None)
re_encoder_grads_list.insert(0, None)
observed_arr, encoded_arr, decoded_arr, re_encoded_arr = self.__inferenced_tuple
delta_arr = self.__encoder_decoder_controller.computable_loss.compute_delta(
decoded_arr,
observed_arr
)
delta_arr[:, -1] += re_encoder_delta_arr[:, -1]
decoder_grads_list, encoder_delta_arr, encoder_grads_list = self.__encoder_decoder_controller.back_propagation(
delta_arr
)
return re_encoder_grads_list, decoder_grads_list, encoder_delta_arr, encoder_grads_list
[docs] def optimize(
self,
re_encoder_grads_list,
decoder_grads_list,
encoder_grads_list,
learning_rate,
epoch
):
'''
Back propagation.
Args:
re_encoder_grads_list: re-encoder's `list` of graduations.
decoder_grads_list: decoder's `list` of graduations.
encoder_grads_list: encoder's `list` of graduations.
learning_rate: Learning rate.
epoch: Now epoch.
'''
self.__retrospective_encoder.optimize(re_encoder_grads_list, learning_rate, epoch)
self.__encoder_decoder_controller.optimize(
decoder_grads_list,
encoder_grads_list,
learning_rate,
epoch
)
[docs] def compute_retrospective_loss(self):
'''
Compute retrospective loss.
Returns:
The tuple data.
- `np.ndarray` of delta.
- `np.ndarray` of losses of each batch.
- float of loss of all batch.
'''
observed_arr, encoded_arr, decoded_arr, re_encoded_arr = self.__inferenced_tuple
batch_size = observed_arr.shape[0]
if self.__input_neuron_count == self.__hidden_neuron_count:
target_arr = encoded_arr - np.expand_dims(observed_arr.mean(axis=2), axis=2)
summary_delta_arr = np.sqrt(np.power(decoded_arr - target_arr, 2))
else:
# For each batch, draw a samples from the Uniform distribution.
if self.__input_neuron_count > self.__hidden_neuron_count:
all_dim_arr = np.arange(self.__input_neuron_count)
np.random.shuffle(all_dim_arr)
choiced_dim_arr = all_dim_arr[:self.__hidden_neuron_count]
target_arr = encoded_arr - np.expand_dims(observed_arr[:, :, choiced_dim_arr].mean(axis=2), axis=2)
summary_delta_arr = np.sqrt(np.power(decoded_arr[:, :, choiced_dim_arr] - target_arr, 2))
else:
all_dim_arr = np.arange(self.__hidden_neuron_count)
np.random.shuffle(all_dim_arr)
choiced_dim_arr = all_dim_arr[:self.__input_neuron_count]
target_arr = encoded_arr[:, :, choiced_dim_arr] - np.expand_dims(observed_arr.mean(axis=2), axis=2)
summary_delta_arr = np.sqrt(np.power(decoded_arr - target_arr, 2))
summary_delta_arr = np.nan_to_num(summary_delta_arr)
summary_delta_arr = (summary_delta_arr - summary_delta_arr.mean()) / (summary_delta_arr.std() + 1e-08)
match_delta_arr = np.sqrt(np.power(encoded_arr[:, -1] - re_encoded_arr[:, -1], 2))
match_delta_arr = np.nan_to_num(match_delta_arr)
match_delta_arr = (match_delta_arr - match_delta_arr.mean()) / (match_delta_arr.std() + 1e-08)
other_encoded_delta_arr = np.nansum(
np.sqrt(
np.power(
np.maximum(
0,
encoded_arr[:, :-1] - re_encoded_arr[:, -1].reshape(
re_encoded_arr[:, -1].shape[0],
1,
re_encoded_arr[:, -1].shape[1]
)
),
2
)
) + self.__margin_param,
axis=1
)
other_encoded_delta_arr = np.nan_to_num(other_encoded_delta_arr)
other_encoded_delta_arr = (other_encoded_delta_arr - other_encoded_delta_arr.mean()) / (other_encoded_delta_arr.std() + 1e-08)
other_re_encoded_delta_arr = np.nansum(
np.sqrt(
np.power(
np.maximum(
0,
encoded_arr[:, -1].reshape(
encoded_arr[:, -1].shape[0],
1,
encoded_arr[:, -1].shape[1]
) - re_encoded_arr[:, :-1],
),
2
)
) + self.__margin_param,
axis=1
)
other_encoded_delta_arr = np.nan_to_num(other_encoded_delta_arr)
other_re_encoded_delta_arr = (other_re_encoded_delta_arr - other_re_encoded_delta_arr.mean()) / (other_re_encoded_delta_arr.std() + 1e-08)
mismatch_delta_arr = (match_delta_arr - other_encoded_delta_arr) + (match_delta_arr - other_re_encoded_delta_arr)
delta_arr = summary_delta_arr + np.expand_dims(self.__retrospective_lambda * match_delta_arr, axis=1) + np.expand_dims(self.__retrospective_eta * mismatch_delta_arr, axis=1)
v = np.linalg.norm(delta_arr)
if v > self.__grad_clip_threshold:
delta_arr = delta_arr * self.__grad_clip_threshold / v
loss = np.square(delta_arr).mean()
loss_arr = np.square(delta_arr).sum(axis=1).mean(axis=1)
return delta_arr, loss_arr, loss
def __change_inferencing_mode(self, inferencing_mode):
'''
Change dropout rate in Encoder/Decoder.
Args:
dropout_rate: The probalibity of dropout.
'''
self.__encoder_decoder_controller.decoder.opt_params.inferencing_mode = inferencing_mode
self.__encoder_decoder_controller.encoder.opt_params.inferencing_mode = inferencing_mode
self.__retrospective_encoder.opt_params.inferencing_mode = inferencing_mode
def __remember_best_params(self, encoder_best_params_list, decoder_best_params_list, re_encoder_best_params_list):
'''
Remember best parameters.
Args:
encoder_best_params_list: `list` of encoder's parameters.
decoder_best_params_list: `list` of decoder's parameters.
re_encoder_best_params_list: `list` of re-decoder's parameters.
'''
if len(encoder_best_params_list) > 0 and len(decoder_best_params_list) > 0:
self.__encoder_decoder_controller.encoder.graph.weights_lstm_hidden_arr = encoder_best_params_list[0]
self.__encoder_decoder_controller.encoder.graph.weights_lstm_observed_arr = encoder_best_params_list[1]
self.__encoder_decoder_controller.encoder.graph.lstm_bias_arr = encoder_best_params_list[2]
self.__encoder_decoder_controller.decoder.graph.weights_lstm_hidden_arr = decoder_best_params_list[0]
self.__encoder_decoder_controller.decoder.graph.weights_lstm_observed_arr = decoder_best_params_list[1]
self.__encoder_decoder_controller.decoder.graph.lstm_bias_arr = decoder_best_params_list[2]
self.__retrospective_encoder.graph.weights_lstm_hidden_arr = re_encoder_best_params_list[0]
self.__retrospective_encoder.graph.weights_lstm_observed_arr = re_encoder_best_params_list[1]
self.__retrospective_encoder.graph.lstm_bias_arr = re_encoder_best_params_list[2]
self.__logger.debug("Best params are saved.")
def __verificate_retrospective_loss(self, train_loss, test_loss):
self.__logger.debug("Epoch: " + str(len(self.__logs_tuple_list) + 1))
self.__logger.debug("Loss: ")
self.__logger.debug(
"Training: " + str(train_loss) + " Test: " + str(test_loss)
)
self.__logs_tuple_list.append(
(
train_loss,
test_loss
)
)
[docs] def get_logs_arr(self):
''' getter '''
return np.array(
self.__logs_tuple_list,
)
[docs] def set_readonly(self, value):
''' setter '''
raise TypeError()
logs_arr = property(get_logs_arr, set_readonly)
[docs] def get_encoder_decoder_controller(self):
''' getter '''
return self.__encoder_decoder_controller
encoder_decoder_controller = property(get_encoder_decoder_controller, set_readonly)
[docs] def get_retrospective_encoder(self):
''' getter '''
return self.__retrospective_encoder
retrospective_encoder = property(get_retrospective_encoder, set_readonly)