From 38c4afef8cf4c12c03404d0ede4421a60e3ef6b7 Mon Sep 17 00:00:00 2001 From: Tobias Wochinger Date: Tue, 21 Apr 2020 14:38:30 +0200 Subject: [PATCH 01/18] make timeout for rasa shell configurable --- changelog/4606.improvement.rst | 4 ++++ rasa/core/channels/console.py | 14 +++++++++++++- tests/core/test_channels.py | 14 +++++++++++--- 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 changelog/4606.improvement.rst diff --git a/changelog/4606.improvement.rst b/changelog/4606.improvement.rst new file mode 100644 index 000000000000..525adc6379a5 --- /dev/null +++ b/changelog/4606.improvement.rst @@ -0,0 +1,4 @@ +The stream reading timeout for ``rasa shell` is now configurable by using the +environment variable ``RASA_SHELL_STREAM_READING_TIMEOUT_IN_SECONDS``. +This can help to fix problems when using ``rasa shell`` with custom actions which run +10 seconds or longer. diff --git a/rasa/core/channels/console.py b/rasa/core/channels/console.py index 2fd9b4d8bf13..ab1437635fcc 100644 --- a/rasa/core/channels/console.py +++ b/rasa/core/channels/console.py @@ -2,6 +2,7 @@ import json import logging import asyncio +import os from typing import Text, Optional, Dict, List import aiohttp @@ -19,6 +20,7 @@ logger = logging.getLogger(__name__) +STREAM_READING_TIMEOUT_ENV = "RASA_SHELL_STREAM_READING_TIMEOUT_IN_SECONDS" DEFAULT_STREAM_READING_TIMEOUT_IN_SECONDS = 10 @@ -97,7 +99,7 @@ async def send_message_receive_stream( url = f"{server_url}/webhooks/rest/webhook?stream=true&token={auth_token}" # Define timeout to not keep reading in case the server crashed in between - timeout = ClientTimeout(DEFAULT_STREAM_READING_TIMEOUT_IN_SECONDS) + timeout = _get_stream_reading_timeout() async with aiohttp.ClientSession(timeout=timeout) as session: async with session.post(url, json=payload, raise_for_status=True) as resp: @@ -107,6 +109,16 @@ async def send_message_receive_stream( yield json.loads(line.decode(DEFAULT_ENCODING)) +def _get_stream_reading_timeout() -> ClientTimeout: + timeout_in_seconds = int( + os.environ.get( + STREAM_READING_TIMEOUT_ENV, DEFAULT_STREAM_READING_TIMEOUT_IN_SECONDS + ) + ) + + return ClientTimeout(timeout_in_seconds) + + async def record_messages( sender_id, server_url=DEFAULT_SERVER_URL, diff --git a/tests/core/test_channels.py b/tests/core/test_channels.py index 35d331bc1606..2b1d249c6e60 100644 --- a/tests/core/test_channels.py +++ b/tests/core/test_channels.py @@ -1,16 +1,17 @@ import json import logging -import urllib.parse from typing import Dict from unittest.mock import patch, MagicMock, Mock import pytest +from _pytest.monkeypatch import MonkeyPatch +from aiohttp import ClientTimeout from aioresponses import aioresponses from sanic import Sanic import rasa.core.run from rasa.core import utils -from rasa.core.channels import RasaChatInput +from rasa.core.channels import RasaChatInput, console from rasa.core.channels.channel import UserMessage from rasa.core.channels.rasa_chat import ( JWT_USERNAME_KEY, @@ -548,7 +549,7 @@ def test_slack_metadata_missing_keys(): "channel": channel, "event_ts": "1579802617.000800", "channel_type": "im", - }, + } } input_channel = SlackInput( @@ -1032,3 +1033,10 @@ def test_has_user_permission_to_send_messages_to_conversation_without_permission assert not RasaChatInput._has_user_permission_to_send_messages_to_conversation( jwt, message ) + + +def test_set_console_stream_reading_timeout(monkeypatch: MonkeyPatch): + expected = 100 + monkeypatch.setenv(console.STREAM_READING_TIMEOUT_ENV, str(100)) + + assert console._get_stream_reading_timeout() == ClientTimeout(expected) From 5fde6ed64df806965ae4b9b91097aea3f68576ce Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Tue, 21 Apr 2020 18:04:26 +0200 Subject: [PATCH 02/18] add whatsapp+ prefix to twilio creds --- docs/user-guide/connectors/twilio.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/connectors/twilio.rst b/docs/user-guide/connectors/twilio.rst index 5f5195170e62..13a00aad1326 100644 --- a/docs/user-guide/connectors/twilio.rst +++ b/docs/user-guide/connectors/twilio.rst @@ -39,7 +39,10 @@ Connecting to WhatsApp You can deploy a Rasa Open Source assistant to WhatsApp through Twilio. However, to do so, you have to have a `WhatsApp Business `_ profile. Associate your Whatsapp Business profile with the phone number you purchased through Twilio to -access the `Twilio API for WhatsApp `_. +access the `Twilio API for WhatsApp `_. + +Note that according to the `API documentation `_, +the phone number you use should be prefixed with `whatsapp+` in the ``credentials.yml`` described below. Applying the Credentials @@ -52,7 +55,7 @@ Add the Twilio credentials to your ``credentials.yml``: twilio: account_sid: "ACbc2dxxxxxxxxxxxx19d54bdcd6e41186" auth_token: "e231c197493a7122d475b4xxxxxxxxxx" - twilio_number: "+440123456789" + twilio_number: "+440123456789" # if using WhatsApp: "whatsapp:+440123456789" Make sure to restart your Rasa Open Source server or container to make changes to which connectors are available. From d2a6a1cc51f2d36f35a98c0f7398ad881421f741 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Thu, 23 Apr 2020 16:52:24 +0200 Subject: [PATCH 03/18] Apply suggestions from code review Co-Authored-By: Akela Drissner-Schmid <32450038+akelad@users.noreply.github.com> --- docs/user-guide/connectors/twilio.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user-guide/connectors/twilio.rst b/docs/user-guide/connectors/twilio.rst index 13a00aad1326..ff3b8945ea73 100644 --- a/docs/user-guide/connectors/twilio.rst +++ b/docs/user-guide/connectors/twilio.rst @@ -41,8 +41,8 @@ to have a `WhatsApp Business `_ profile. Ass your Whatsapp Business profile with the phone number you purchased through Twilio to access the `Twilio API for WhatsApp `_. -Note that according to the `API documentation `_, -the phone number you use should be prefixed with `whatsapp+` in the ``credentials.yml`` described below. +According to the `Twilio API documentation `_, +the phone number you use should be prefixed with `whatsapp:` in the ``credentials.yml`` described below. Applying the Credentials From 50536ac509b4414d61e3ba3147fa6d4208ba7969 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Thu, 23 Apr 2020 17:52:47 +0200 Subject: [PATCH 04/18] Revert "Mask vs sequence" --- rasa/nlu/classifiers/diet_classifier.py | 51 +++++++++---------------- rasa/nlu/selectors/response_selector.py | 28 +++++--------- rasa/utils/tensorflow/model_data.py | 11 ++++-- 3 files changed, 36 insertions(+), 54 deletions(-) diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 96bacf9b6dda..67e14368740a 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -87,10 +87,10 @@ TEXT_FEATURES = f"{TEXT}_features" LABEL_FEATURES = f"{LABEL}_features" +TEXT_MASK = f"{TEXT}_mask" +LABEL_MASK = f"{LABEL}_mask" LABEL_IDS = f"{LABEL}_ids" TAG_IDS = "tag_ids" -TEXT_SEQ_LENGTH = f"{TEXT}_lengths" -LABEL_SEQ_LENGTH = f"{LABEL}_lengths" class DIETClassifier(IntentClassifier, EntityExtractor): @@ -484,7 +484,7 @@ def _create_label_data( # to track correctly dynamic sequences label_data.add_features(LABEL_IDS, [np.expand_dims(label_ids, -1)]) - label_data.add_lengths(LABEL_SEQ_LENGTH, LABEL_FEATURES) + label_data.add_mask(LABEL_MASK, LABEL_FEATURES) return label_data @@ -558,8 +558,8 @@ def _create_model_data( model_data.add_features(LABEL_IDS, [np.expand_dims(label_ids, -1)]) model_data.add_features(TAG_IDS, [tag_ids]) - model_data.add_lengths(TEXT_SEQ_LENGTH, TEXT_FEATURES) - model_data.add_lengths(LABEL_SEQ_LENGTH, LABEL_FEATURES) + model_data.add_mask(TEXT_MASK, TEXT_FEATURES) + model_data.add_mask(LABEL_MASK, LABEL_FEATURES) return model_data @@ -1165,6 +1165,10 @@ def _prepare_entity_recognition_layers(self) -> None: average="micro", ) + @staticmethod + def _get_sequence_lengths(mask: tf.Tensor) -> tf.Tensor: + return tf.cast(tf.reduce_sum(mask[:, :, 0], axis=1), tf.int32) + def _combine_sparse_dense_features( self, features: List[Union[np.ndarray, tf.Tensor, tf.SparseTensor]], @@ -1255,14 +1259,10 @@ def _create_sequence( def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: all_label_ids = self.tf_label_data[LABEL_IDS][0] - - label_lengths = self.sequence_lengths_for( - self.tf_label_data[LABEL_SEQ_LENGTH][0] - ) - mask_label = self._compute_mask(label_lengths) - x = self._create_bow( - self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name, + self.tf_label_data[LABEL_FEATURES], + self.tf_label_data[LABEL_MASK][0], + self.label_name, ) all_labels_embed = self._tf_layers[f"embed.{LABEL}"](x) @@ -1356,23 +1356,13 @@ def _calculate_entity_loss( return loss, f1 - @staticmethod - def _compute_mask(sequence_lengths: tf.Tensor) -> tf.Tensor: - mask = tf.sequence_mask(sequence_lengths, dtype=tf.float32) - # explicitly add last dimension to mask - # to track correctly dynamic sequences - return tf.expand_dims(mask, -1) - - def sequence_lengths_for(self, sequence_lengths: tf.Tensor) -> tf.Tensor: - return tf.cast(sequence_lengths, dtype=tf.int32) - def batch_loss( self, batch_in: Union[Tuple[tf.Tensor], Tuple[np.ndarray]] ) -> tf.Tensor: tf_batch_data = self.batch_to_model_data_format(batch_in, self.data_signature) - sequence_lengths = self.sequence_lengths_for(tf_batch_data[TEXT_SEQ_LENGTH][0]) - mask_text = self._compute_mask(sequence_lengths) + mask_text = tf_batch_data[TEXT_MASK][0] + sequence_lengths = self._get_sequence_lengths(mask_text) ( text_transformed, @@ -1401,14 +1391,11 @@ def batch_loss( # get _cls_ vector for intent classification cls = self._last_token(text_transformed, sequence_lengths) - label_lengths = self.sequence_lengths_for( - tf_batch_data[LABEL_SEQ_LENGTH][0] - ) - mask_label = self._compute_mask(label_lengths) - label_ids = tf_batch_data[LABEL_IDS][0] label = self._create_bow( - tf_batch_data[LABEL_FEATURES], mask_label, self.label_name, + tf_batch_data[LABEL_FEATURES], + tf_batch_data[LABEL_MASK][0], + self.label_name, ) loss, acc = self._calculate_label_loss(cls, label, label_ids) self.intent_loss.update_state(loss) @@ -1434,8 +1421,8 @@ def batch_predict( batch_in, self.predict_data_signature ) - sequence_lengths = self.sequence_lengths_for(tf_batch_data[TEXT_SEQ_LENGTH][0]) - mask_text = self._compute_mask(sequence_lengths) + mask_text = tf_batch_data[TEXT_MASK][0] + sequence_lengths = self._get_sequence_lengths(mask_text) text_transformed, _, _, _ = self._create_sequence( tf_batch_data[TEXT_FEATURES], mask_text, self.text_name diff --git a/rasa/nlu/selectors/response_selector.py b/rasa/nlu/selectors/response_selector.py index 75ff9815ac26..b8b4c6081228 100644 --- a/rasa/nlu/selectors/response_selector.py +++ b/rasa/nlu/selectors/response_selector.py @@ -17,9 +17,9 @@ DIET, TEXT_FEATURES, LABEL_FEATURES, + TEXT_MASK, + LABEL_MASK, LABEL_IDS, - TEXT_SEQ_LENGTH, - LABEL_SEQ_LENGTH, ) from rasa.utils.tensorflow.constants import ( LABEL, @@ -432,10 +432,8 @@ def _prepare_layers(self) -> None: def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: all_label_ids = self.tf_label_data[LABEL_IDS][0] - sequence_lengths_label = self.sequence_lengths_for( - self.tf_label_data[LABEL_SEQ_LENGTH][0] - ) - mask_label = self._compute_mask(sequence_lengths_label) + mask_label = self.tf_label_data[LABEL_MASK][0] + sequence_lengths_label = self._get_sequence_lengths(mask_label) label_transformed, _, _, _ = self._create_sequence( self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name @@ -451,10 +449,8 @@ def batch_loss( ) -> tf.Tensor: tf_batch_data = self.batch_to_model_data_format(batch_in, self.data_signature) - sequence_lengths_text = self.sequence_lengths_for( - tf_batch_data[TEXT_SEQ_LENGTH][0] - ) - mask_text = self._compute_mask(sequence_lengths_text) + mask_text = tf_batch_data[TEXT_MASK][0] + sequence_lengths_text = self._get_sequence_lengths(mask_text) ( text_transformed, @@ -469,10 +465,8 @@ def batch_loss( sequence_ids=True, ) - sequence_lengths_label = self.sequence_lengths_for( - tf_batch_data[LABEL_SEQ_LENGTH][0] - ) - mask_label = self._compute_mask(sequence_lengths_label) + mask_label = tf_batch_data[LABEL_MASK][0] + sequence_lengths_label = self._get_sequence_lengths(mask_label) label_transformed, _, _, _ = self._create_sequence( tf_batch_data[LABEL_FEATURES], mask_label, self.label_name @@ -512,10 +506,8 @@ def batch_predict( batch_in, self.predict_data_signature ) - sequence_lengths_text = self.sequence_lengths_for( - tf_batch_data[TEXT_SEQ_LENGTH][0] - ) - mask_text = self._compute_mask(sequence_lengths_text) + mask_text = tf_batch_data[TEXT_MASK][0] + sequence_lengths_text = self._get_sequence_lengths(mask_text) text_transformed, _, _, _ = self._create_sequence( tf_batch_data[TEXT_FEATURES], mask_text, self.text_name diff --git a/rasa/utils/tensorflow/model_data.py b/rasa/utils/tensorflow/model_data.py index a233cbc4cc6b..0fbdba4e9d86 100644 --- a/rasa/utils/tensorflow/model_data.py +++ b/rasa/utils/tensorflow/model_data.py @@ -144,8 +144,9 @@ def add_features(self, key: Text, features: List[np.ndarray]): # update number of examples self.num_examples = self.number_of_examples() - def add_lengths(self, key: Text, from_key: Text) -> None: - """Adds np.array of lengths of sequences to data under given key.""" + def add_mask(self, key: Text, from_key: Text): + """Calculate mask for given key and put it under specified key.""" + if not self.data.get(from_key): return @@ -153,8 +154,10 @@ def add_lengths(self, key: Text, from_key: Text) -> None: for data in self.data[from_key]: if data.size > 0: - lengths = np.array([x.shape[0] for x in data]) - self.data[key].append(lengths) + # explicitly add last dimension to mask + # to track correctly dynamic sequences + mask = np.array([np.ones((x.shape[0], 1)) for x in data]) + self.data[key].append(mask) break def split( From a1a688756d832a1b33a9531c8f6980218bb9674f Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Thu, 23 Apr 2020 17:53:24 +0200 Subject: [PATCH 05/18] Revert "Move activation to transformer block" --- rasa/nlu/classifiers/diet_classifier.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 96bacf9b6dda..2d5414d9421b 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -1246,10 +1246,7 @@ def _create_sequence( outputs = self._tf_layers[f"{name}_transformer"]( transformer_inputs, 1 - mask, self._training ) - - if self.config[NUM_TRANSFORMER_LAYERS] > 0: - # apply activation - outputs = tfa.activations.gelu(outputs) + outputs = tfa.activations.gelu(outputs) return outputs, inputs, seq_ids, lm_mask_bool From 5915f5e9c824b8c3516e26d9da3cea44fb0ca6c7 Mon Sep 17 00:00:00 2001 From: Ella Date: Thu, 23 Apr 2020 18:02:24 +0200 Subject: [PATCH 06/18] changelog entry --- changelog/5709.bugfix.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/5709.bugfix.rst diff --git a/changelog/5709.bugfix.rst b/changelog/5709.bugfix.rst new file mode 100644 index 000000000000..758ba07ce7d6 --- /dev/null +++ b/changelog/5709.bugfix.rst @@ -0,0 +1,4 @@ +Reverted changes in 1.9.6 that led to model incompatibility. Upgrade to 1.9.7 to fix +``self.sequence_lengths_for(tf_batch_data[TEXT_SEQ_LENGTH][0]) +IndexError: list index out of range`` +error without needing to retrain earlier 1.9 models. From c0b3065fb0279aaffe44b722aa2093164e3e45ef Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Thu, 23 Apr 2020 18:04:22 +0200 Subject: [PATCH 07/18] Update 5709.bugfix.rst --- changelog/5709.bugfix.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/changelog/5709.bugfix.rst b/changelog/5709.bugfix.rst index 758ba07ce7d6..ba9d2198028e 100644 --- a/changelog/5709.bugfix.rst +++ b/changelog/5709.bugfix.rst @@ -1,4 +1,6 @@ Reverted changes in 1.9.6 that led to model incompatibility. Upgrade to 1.9.7 to fix -``self.sequence_lengths_for(tf_batch_data[TEXT_SEQ_LENGTH][0]) -IndexError: list index out of range`` +``self.sequence_lengths_for(tf_batch_data[TEXT_SEQ_LENGTH][0]) IndexError: list index out of range`` error without needing to retrain earlier 1.9 models. + +Therefore, all 1.9 models `except for 1.9.6` will be compatible; a model trained on 1.9.6 will need +to be retrained on 1.9.7. From 976aa76c29e0a353d6a4af4af7ff2d7c656d5a17 Mon Sep 17 00:00:00 2001 From: Ella Date: Thu, 23 Apr 2020 20:12:14 +0200 Subject: [PATCH 08/18] prepared release of version 1.9.7 --- CHANGELOG.rst | 24 ++++++++++++++++++++++++ changelog/4606.improvement.rst | 4 ---- changelog/5631.misc.rst | 1 - changelog/5709.bugfix.rst | 6 ------ pyproject.toml | 2 +- rasa/version.py | 2 +- 6 files changed, 26 insertions(+), 13 deletions(-) delete mode 100644 changelog/4606.improvement.rst delete mode 100644 changelog/5631.misc.rst delete mode 100644 changelog/5709.bugfix.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aaaa78441941..9fac1be86c93 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,30 @@ This project adheres to `Semantic Versioning`_ starting with version 1.0. .. towncrier release notes start +[1.9.7] - 2020-04-23 +^^^^^^^^^^^^^^^^^^^^ + +Improvements +------------ +- `#4606 `_: The stream reading timeout for ``rasa shell` is now configurable by using the + environment variable ``RASA_SHELL_STREAM_READING_TIMEOUT_IN_SECONDS``. + This can help to fix problems when using ``rasa shell`` with custom actions which run + 10 seconds or longer. + +Bugfixes +-------- +- `#5709 `_: Reverted changes in 1.9.6 that led to model incompatibility. Upgrade to 1.9.7 to fix + ``self.sequence_lengths_for(tf_batch_data[TEXT_SEQ_LENGTH][0]) IndexError: list index out of range`` + error without needing to retrain earlier 1.9 models. + + Therefore, all 1.9 models `except for 1.9.6` will be compatible; a model trained on 1.9.6 will need + to be retrained on 1.9.7. + +Miscellaneous internal changes +------------------------------ +- #5631 + + [1.9.6] - 2020-04-15 ^^^^^^^^^^^^^^^^^^^^ diff --git a/changelog/4606.improvement.rst b/changelog/4606.improvement.rst deleted file mode 100644 index 525adc6379a5..000000000000 --- a/changelog/4606.improvement.rst +++ /dev/null @@ -1,4 +0,0 @@ -The stream reading timeout for ``rasa shell` is now configurable by using the -environment variable ``RASA_SHELL_STREAM_READING_TIMEOUT_IN_SECONDS``. -This can help to fix problems when using ``rasa shell`` with custom actions which run -10 seconds or longer. diff --git a/changelog/5631.misc.rst b/changelog/5631.misc.rst deleted file mode 100644 index d1994407f85c..000000000000 --- a/changelog/5631.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Set regularization constant in ``EmbeddingIntentClassifier`` to 0.001. \ No newline at end of file diff --git a/changelog/5709.bugfix.rst b/changelog/5709.bugfix.rst deleted file mode 100644 index ba9d2198028e..000000000000 --- a/changelog/5709.bugfix.rst +++ /dev/null @@ -1,6 +0,0 @@ -Reverted changes in 1.9.6 that led to model incompatibility. Upgrade to 1.9.7 to fix -``self.sequence_lengths_for(tf_batch_data[TEXT_SEQ_LENGTH][0]) IndexError: list index out of range`` -error without needing to retrain earlier 1.9 models. - -Therefore, all 1.9 models `except for 1.9.6` will be compatible; a model trained on 1.9.6 will need -to be retrained on 1.9.7. diff --git a/pyproject.toml b/pyproject.toml index 99b12d03c191..cedd8bbc6d2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ exclude = "((.eggs | .git | .pytype | .pytest_cache | build | dist))" [tool.poetry] name = "rasa" -version = "1.9.6" +version = "1.9.7" description = "Open source machine learning framework to automate text- and voice-based conversations: NLU, dialogue management, connect to Slack, Facebook, and more - Create chatbots and voice assistants" authors = [ "Rasa Technologies GmbH ",] maintainers = [ "Tom Bocklisch ",] diff --git a/rasa/version.py b/rasa/version.py index 2f6bbbacee47..aa3cab48f5d3 100644 --- a/rasa/version.py +++ b/rasa/version.py @@ -1,3 +1,3 @@ # this file will automatically be changed, # do not add anything but the version number here! -__version__ = "1.9.6" +__version__ = "1.9.7" From ecb797399fa867576aa0c09ea1e6545e450e26ab Mon Sep 17 00:00:00 2001 From: Zhenya Razumovskaia Date: Thu, 2 Apr 2020 07:43:12 -0400 Subject: [PATCH 09/18] merge conflict --- rasa/nlu/classifiers/diet_classifier.py | 27 ++++++++++++++++--------- rasa/nlu/selectors/response_selector.py | 24 ++++++++++++++-------- rasa/utils/tensorflow/model_data.py | 14 +++++++++++++ 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 64579161602f..4d33b2c738f7 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -1256,10 +1256,12 @@ def _create_sequence( def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: all_label_ids = self.tf_label_data[LABEL_IDS][0] + + label_lengths = tf.cast(self.tf_label_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32) + mask_label = self.compute_mask(label_lengths) + x = self._create_bow( - self.tf_label_data[LABEL_FEATURES], - self.tf_label_data[LABEL_MASK][0], - self.label_name, + self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name, ) all_labels_embed = self._tf_layers[f"embed.{LABEL}"](x) @@ -1353,13 +1355,17 @@ def _calculate_entity_loss( return loss, f1 + def compute_mask(self, sequence_lengths): + mask = tf.sequence_mask(sequence_lengths, dtype=tf.float32) + return tf.expand_dims(mask, -1) + def batch_loss( self, batch_in: Union[Tuple[tf.Tensor], Tuple[np.ndarray]] ) -> tf.Tensor: tf_batch_data = self.batch_to_model_data_format(batch_in, self.data_signature) - mask_text = tf_batch_data[TEXT_MASK][0] - sequence_lengths = self._get_sequence_lengths(mask_text) + sequence_lengths = tf.cast(tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32) + mask_text = self.compute_mask(sequence_lengths) ( text_transformed, @@ -1388,11 +1394,12 @@ def batch_loss( # get _cls_ vector for intent classification cls = self._last_token(text_transformed, sequence_lengths) + label_lengths = tf.cast(tf_batch_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32) + mask_label = self.compute_mask(label_lengths) + label_ids = tf_batch_data[LABEL_IDS][0] label = self._create_bow( - tf_batch_data[LABEL_FEATURES], - tf_batch_data[LABEL_MASK][0], - self.label_name, + tf_batch_data[LABEL_FEATURES], mask_label, self.label_name, ) loss, acc = self._calculate_label_loss(cls, label, label_ids) self.intent_loss.update_state(loss) @@ -1418,8 +1425,8 @@ def batch_predict( batch_in, self.predict_data_signature ) - mask_text = tf_batch_data[TEXT_MASK][0] - sequence_lengths = self._get_sequence_lengths(mask_text) + sequence_lengths = tf.cast(tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32) + mask_text = self.compute_mask(sequence_lengths) text_transformed, _, _, _ = self._create_sequence( tf_batch_data[TEXT_FEATURES], mask_text, self.text_name diff --git a/rasa/nlu/selectors/response_selector.py b/rasa/nlu/selectors/response_selector.py index b8b4c6081228..9c4c46bddaa4 100644 --- a/rasa/nlu/selectors/response_selector.py +++ b/rasa/nlu/selectors/response_selector.py @@ -432,8 +432,10 @@ def _prepare_layers(self) -> None: def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: all_label_ids = self.tf_label_data[LABEL_IDS][0] - mask_label = self.tf_label_data[LABEL_MASK][0] - sequence_lengths_label = self._get_sequence_lengths(mask_label) + sequence_lengths_label = tf.cast( + self.tf_label_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32 + ) + mask_label = self.compute_mask(sequence_lengths_label) label_transformed, _, _, _ = self._create_sequence( self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name @@ -449,8 +451,10 @@ def batch_loss( ) -> tf.Tensor: tf_batch_data = self.batch_to_model_data_format(batch_in, self.data_signature) - mask_text = tf_batch_data[TEXT_MASK][0] - sequence_lengths_text = self._get_sequence_lengths(mask_text) + sequence_lengths_text = tf.cast( + tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32 + ) + mask_text = self.compute_mask(sequence_lengths_text) ( text_transformed, @@ -465,8 +469,10 @@ def batch_loss( sequence_ids=True, ) - mask_label = tf_batch_data[LABEL_MASK][0] - sequence_lengths_label = self._get_sequence_lengths(mask_label) + sequence_lengths_label = tf.cast( + tf_batch_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32 + ) + mask_label = self.compute_mask(sequence_lengths_label) label_transformed, _, _, _ = self._create_sequence( tf_batch_data[LABEL_FEATURES], mask_label, self.label_name @@ -506,8 +512,10 @@ def batch_predict( batch_in, self.predict_data_signature ) - mask_text = tf_batch_data[TEXT_MASK][0] - sequence_lengths_text = self._get_sequence_lengths(mask_text) + sequence_lengths_text = tf.cast( + tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32 + ) + mask_text = self.compute_mask(sequence_lengths_text) text_transformed, _, _, _ = self._create_sequence( tf_batch_data[TEXT_FEATURES], mask_text, self.text_name diff --git a/rasa/utils/tensorflow/model_data.py b/rasa/utils/tensorflow/model_data.py index 0fbdba4e9d86..a6ae957a8f98 100644 --- a/rasa/utils/tensorflow/model_data.py +++ b/rasa/utils/tensorflow/model_data.py @@ -160,6 +160,20 @@ def add_mask(self, key: Text, from_key: Text): self.data[key].append(mask) break + def add_lengths(self, key: Text, from_key: Text): + if not self.data.get(from_key): + return + + self.data[key] = [] + + for data in self.data[from_key]: + if data.size > 0: + # explicitly add last dimension to mask + # to track correctly dynamic sequences + lengths = np.array([x.shape[0] for x in data]) + self.data[key].append(lengths) + break + def split( self, number_of_test_examples: int, random_seed: int ) -> Tuple["RasaModelData", "RasaModelData"]: From 036567a28cb1e7d8fcc13582a31e2d1c0ff36994 Mon Sep 17 00:00:00 2001 From: Zhenya Razumovskaia Date: Thu, 2 Apr 2020 09:12:47 -0400 Subject: [PATCH 10/18] small changes --- rasa/nlu/classifiers/diet_classifier.py | 10 +++++----- rasa/nlu/selectors/response_selector.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 4d33b2c738f7..5f00bd289002 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -87,10 +87,10 @@ TEXT_FEATURES = f"{TEXT}_features" LABEL_FEATURES = f"{LABEL}_features" -TEXT_MASK = f"{TEXT}_mask" -LABEL_MASK = f"{LABEL}_mask" LABEL_IDS = f"{LABEL}_ids" TAG_IDS = "tag_ids" +TEXT_SEQ_LENGTH = f"{TEXT}_lengths" +LABEL_SEQ_LENGTH = f"{LABEL}_lengths" class DIETClassifier(IntentClassifier, EntityExtractor): @@ -484,7 +484,7 @@ def _create_label_data( # to track correctly dynamic sequences label_data.add_features(LABEL_IDS, [np.expand_dims(label_ids, -1)]) - label_data.add_mask(LABEL_MASK, LABEL_FEATURES) + label_data.add_lengths(LABEL_SEQ_LENGTH, LABEL_FEATURES) return label_data @@ -558,8 +558,8 @@ def _create_model_data( model_data.add_features(LABEL_IDS, [np.expand_dims(label_ids, -1)]) model_data.add_features(TAG_IDS, [tag_ids]) - model_data.add_mask(TEXT_MASK, TEXT_FEATURES) - model_data.add_mask(LABEL_MASK, LABEL_FEATURES) + model_data.add_lengths(TEXT_SEQ_LENGTH, TEXT_FEATURES) + model_data.add_lengths(LABEL_SEQ_LENGTH, LABEL_FEATURES) return model_data diff --git a/rasa/nlu/selectors/response_selector.py b/rasa/nlu/selectors/response_selector.py index 9c4c46bddaa4..91d924b747ab 100644 --- a/rasa/nlu/selectors/response_selector.py +++ b/rasa/nlu/selectors/response_selector.py @@ -17,9 +17,9 @@ DIET, TEXT_FEATURES, LABEL_FEATURES, - TEXT_MASK, - LABEL_MASK, LABEL_IDS, + TEXT_SEQ_LENGTH, + LABEL_SEQ_LENGTH, ) from rasa.utils.tensorflow.constants import ( LABEL, From 1cea69aeb8682a8d676975a3aecbb3650f1e470a Mon Sep 17 00:00:00 2001 From: Zhenya Razumovskaia Date: Thu, 2 Apr 2020 11:33:45 -0400 Subject: [PATCH 11/18] small changed by comments --- rasa/nlu/classifiers/diet_classifier.py | 16 +++++++--------- rasa/nlu/selectors/response_selector.py | 8 ++++---- rasa/utils/tensorflow/model_data.py | 21 +++------------------ 3 files changed, 14 insertions(+), 31 deletions(-) diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 5f00bd289002..25e5c894dea2 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -1165,10 +1165,6 @@ def _prepare_entity_recognition_layers(self) -> None: average="micro", ) - @staticmethod - def _get_sequence_lengths(mask: tf.Tensor) -> tf.Tensor: - return tf.cast(tf.reduce_sum(mask[:, :, 0], axis=1), tf.int32) - def _combine_sparse_dense_features( self, features: List[Union[np.ndarray, tf.Tensor, tf.SparseTensor]], @@ -1258,7 +1254,7 @@ def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: all_label_ids = self.tf_label_data[LABEL_IDS][0] label_lengths = tf.cast(self.tf_label_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32) - mask_label = self.compute_mask(label_lengths) + mask_label = self._compute_mask(label_lengths) x = self._create_bow( self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name, @@ -1355,8 +1351,10 @@ def _calculate_entity_loss( return loss, f1 - def compute_mask(self, sequence_lengths): + def _compute_mask(self, sequence_lengths: tf.Tensor) -> tf.Tensor: mask = tf.sequence_mask(sequence_lengths, dtype=tf.float32) + # explicitly add last dimension to mask + # to track correctly dynamic sequences return tf.expand_dims(mask, -1) def batch_loss( @@ -1365,7 +1363,7 @@ def batch_loss( tf_batch_data = self.batch_to_model_data_format(batch_in, self.data_signature) sequence_lengths = tf.cast(tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32) - mask_text = self.compute_mask(sequence_lengths) + mask_text = self._compute_mask(sequence_lengths) ( text_transformed, @@ -1395,7 +1393,7 @@ def batch_loss( cls = self._last_token(text_transformed, sequence_lengths) label_lengths = tf.cast(tf_batch_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32) - mask_label = self.compute_mask(label_lengths) + mask_label = self._compute_mask(label_lengths) label_ids = tf_batch_data[LABEL_IDS][0] label = self._create_bow( @@ -1426,7 +1424,7 @@ def batch_predict( ) sequence_lengths = tf.cast(tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32) - mask_text = self.compute_mask(sequence_lengths) + mask_text = self._compute_mask(sequence_lengths) text_transformed, _, _, _ = self._create_sequence( tf_batch_data[TEXT_FEATURES], mask_text, self.text_name diff --git a/rasa/nlu/selectors/response_selector.py b/rasa/nlu/selectors/response_selector.py index 91d924b747ab..36f4318c22e0 100644 --- a/rasa/nlu/selectors/response_selector.py +++ b/rasa/nlu/selectors/response_selector.py @@ -435,7 +435,7 @@ def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: sequence_lengths_label = tf.cast( self.tf_label_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32 ) - mask_label = self.compute_mask(sequence_lengths_label) + mask_label = self._compute_mask(sequence_lengths_label) label_transformed, _, _, _ = self._create_sequence( self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name @@ -454,7 +454,7 @@ def batch_loss( sequence_lengths_text = tf.cast( tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32 ) - mask_text = self.compute_mask(sequence_lengths_text) + mask_text = self._compute_mask(sequence_lengths_text) ( text_transformed, @@ -472,7 +472,7 @@ def batch_loss( sequence_lengths_label = tf.cast( tf_batch_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32 ) - mask_label = self.compute_mask(sequence_lengths_label) + mask_label = self._compute_mask(sequence_lengths_label) label_transformed, _, _, _ = self._create_sequence( tf_batch_data[LABEL_FEATURES], mask_label, self.label_name @@ -515,7 +515,7 @@ def batch_predict( sequence_lengths_text = tf.cast( tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32 ) - mask_text = self.compute_mask(sequence_lengths_text) + mask_text = self._compute_mask(sequence_lengths_text) text_transformed, _, _, _ = self._create_sequence( tf_batch_data[TEXT_FEATURES], mask_text, self.text_name diff --git a/rasa/utils/tensorflow/model_data.py b/rasa/utils/tensorflow/model_data.py index a6ae957a8f98..13b5033e510e 100644 --- a/rasa/utils/tensorflow/model_data.py +++ b/rasa/utils/tensorflow/model_data.py @@ -144,23 +144,10 @@ def add_features(self, key: Text, features: List[np.ndarray]): # update number of examples self.num_examples = self.number_of_examples() - def add_mask(self, key: Text, from_key: Text): - """Calculate mask for given key and put it under specified key.""" - - if not self.data.get(from_key): - return - - self.data[key] = [] - - for data in self.data[from_key]: - if data.size > 0: - # explicitly add last dimension to mask - # to track correctly dynamic sequences - mask = np.array([np.ones((x.shape[0], 1)) for x in data]) - self.data[key].append(mask) - break - def add_lengths(self, key: Text, from_key: Text): + """ + Adds np.array of lengths of sequences to data under given key + """ if not self.data.get(from_key): return @@ -168,8 +155,6 @@ def add_lengths(self, key: Text, from_key: Text): for data in self.data[from_key]: if data.size > 0: - # explicitly add last dimension to mask - # to track correctly dynamic sequences lengths = np.array([x.shape[0] for x in data]) self.data[key].append(lengths) break From 65d4cfd0e940bcb775f7c9053bcda87e17286ac4 Mon Sep 17 00:00:00 2001 From: Zhenya Razumovskaia Date: Thu, 2 Apr 2020 12:06:45 -0400 Subject: [PATCH 12/18] None return type --- rasa/utils/tensorflow/model_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rasa/utils/tensorflow/model_data.py b/rasa/utils/tensorflow/model_data.py index 13b5033e510e..88f7cf28d3b2 100644 --- a/rasa/utils/tensorflow/model_data.py +++ b/rasa/utils/tensorflow/model_data.py @@ -144,7 +144,7 @@ def add_features(self, key: Text, features: List[np.ndarray]): # update number of examples self.num_examples = self.number_of_examples() - def add_lengths(self, key: Text, from_key: Text): + def add_lengths(self, key: Text, from_key: Text) -> None: """ Adds np.array of lengths of sequences to data under given key """ From d6cdc793bcacd53d9861d1edaeda7c3e8f323c00 Mon Sep 17 00:00:00 2001 From: Evgeniia Date: Fri, 3 Apr 2020 03:40:51 -0400 Subject: [PATCH 13/18] Update docstring Co-Authored-By: Tanja --- rasa/utils/tensorflow/model_data.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rasa/utils/tensorflow/model_data.py b/rasa/utils/tensorflow/model_data.py index 88f7cf28d3b2..a233cbc4cc6b 100644 --- a/rasa/utils/tensorflow/model_data.py +++ b/rasa/utils/tensorflow/model_data.py @@ -145,9 +145,7 @@ def add_features(self, key: Text, features: List[np.ndarray]): self.num_examples = self.number_of_examples() def add_lengths(self, key: Text, from_key: Text) -> None: - """ - Adds np.array of lengths of sequences to data under given key - """ + """Adds np.array of lengths of sequences to data under given key.""" if not self.data.get(from_key): return From f25a6cacc3b01e3f77a69e98e2b7965cf9b81d01 Mon Sep 17 00:00:00 2001 From: Zhenya Razumovskaia Date: Fri, 3 Apr 2020 04:40:01 -0400 Subject: [PATCH 14/18] sequence length as a separate method --- rasa/nlu/classifiers/diet_classifier.py | 14 +++++++++----- rasa/nlu/selectors/response_selector.py | 16 ++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 25e5c894dea2..575ac026ce9d 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -1253,7 +1253,7 @@ def _create_sequence( def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: all_label_ids = self.tf_label_data[LABEL_IDS][0] - label_lengths = tf.cast(self.tf_label_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32) + label_lengths = self.sequence_lengths_for(self.tf_label_data[LABEL_SEQ_LENGTH][0]) mask_label = self._compute_mask(label_lengths) x = self._create_bow( @@ -1351,18 +1351,22 @@ def _calculate_entity_loss( return loss, f1 - def _compute_mask(self, sequence_lengths: tf.Tensor) -> tf.Tensor: + @staticmethod + def _compute_mask(sequence_lengths: tf.Tensor) -> tf.Tensor: mask = tf.sequence_mask(sequence_lengths, dtype=tf.float32) # explicitly add last dimension to mask # to track correctly dynamic sequences return tf.expand_dims(mask, -1) + def sequence_lengths_for(self, sequence_lengths: tf.Tensor) -> tf.Tensor: + return tf.cast(sequence_lengths, dtype=tf.int32) + def batch_loss( self, batch_in: Union[Tuple[tf.Tensor], Tuple[np.ndarray]] ) -> tf.Tensor: tf_batch_data = self.batch_to_model_data_format(batch_in, self.data_signature) - sequence_lengths = tf.cast(tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32) + sequence_lengths = self.sequence_lengths_for(tf_batch_data[TEXT_SEQ_LENGTH][0]) mask_text = self._compute_mask(sequence_lengths) ( @@ -1392,7 +1396,7 @@ def batch_loss( # get _cls_ vector for intent classification cls = self._last_token(text_transformed, sequence_lengths) - label_lengths = tf.cast(tf_batch_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32) + label_lengths = self.sequence_lengths_for(tf_batch_data[LABEL_SEQ_LENGTH][0]) mask_label = self._compute_mask(label_lengths) label_ids = tf_batch_data[LABEL_IDS][0] @@ -1423,7 +1427,7 @@ def batch_predict( batch_in, self.predict_data_signature ) - sequence_lengths = tf.cast(tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32) + sequence_lengths = self.sequence_lengths_for(tf_batch_data[TEXT_SEQ_LENGTH][0]) mask_text = self._compute_mask(sequence_lengths) text_transformed, _, _, _ = self._create_sequence( diff --git a/rasa/nlu/selectors/response_selector.py b/rasa/nlu/selectors/response_selector.py index 36f4318c22e0..75ff9815ac26 100644 --- a/rasa/nlu/selectors/response_selector.py +++ b/rasa/nlu/selectors/response_selector.py @@ -432,8 +432,8 @@ def _prepare_layers(self) -> None: def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: all_label_ids = self.tf_label_data[LABEL_IDS][0] - sequence_lengths_label = tf.cast( - self.tf_label_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32 + sequence_lengths_label = self.sequence_lengths_for( + self.tf_label_data[LABEL_SEQ_LENGTH][0] ) mask_label = self._compute_mask(sequence_lengths_label) @@ -451,8 +451,8 @@ def batch_loss( ) -> tf.Tensor: tf_batch_data = self.batch_to_model_data_format(batch_in, self.data_signature) - sequence_lengths_text = tf.cast( - tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32 + sequence_lengths_text = self.sequence_lengths_for( + tf_batch_data[TEXT_SEQ_LENGTH][0] ) mask_text = self._compute_mask(sequence_lengths_text) @@ -469,8 +469,8 @@ def batch_loss( sequence_ids=True, ) - sequence_lengths_label = tf.cast( - tf_batch_data[LABEL_SEQ_LENGTH][0], dtype=tf.int32 + sequence_lengths_label = self.sequence_lengths_for( + tf_batch_data[LABEL_SEQ_LENGTH][0] ) mask_label = self._compute_mask(sequence_lengths_label) @@ -512,8 +512,8 @@ def batch_predict( batch_in, self.predict_data_signature ) - sequence_lengths_text = tf.cast( - tf_batch_data[TEXT_SEQ_LENGTH][0], dtype=tf.int32 + sequence_lengths_text = self.sequence_lengths_for( + tf_batch_data[TEXT_SEQ_LENGTH][0] ) mask_text = self._compute_mask(sequence_lengths_text) From 3d7cc8c9610f4838c5fbc1953e0d9fcb3acec346 Mon Sep 17 00:00:00 2001 From: Zhenya Razumovskaia Date: Fri, 3 Apr 2020 06:12:51 -0400 Subject: [PATCH 15/18] black --- rasa/nlu/classifiers/diet_classifier.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 575ac026ce9d..2d5414d9421b 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -1253,7 +1253,9 @@ def _create_sequence( def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: all_label_ids = self.tf_label_data[LABEL_IDS][0] - label_lengths = self.sequence_lengths_for(self.tf_label_data[LABEL_SEQ_LENGTH][0]) + label_lengths = self.sequence_lengths_for( + self.tf_label_data[LABEL_SEQ_LENGTH][0] + ) mask_label = self._compute_mask(label_lengths) x = self._create_bow( @@ -1396,7 +1398,9 @@ def batch_loss( # get _cls_ vector for intent classification cls = self._last_token(text_transformed, sequence_lengths) - label_lengths = self.sequence_lengths_for(tf_batch_data[LABEL_SEQ_LENGTH][0]) + label_lengths = self.sequence_lengths_for( + tf_batch_data[LABEL_SEQ_LENGTH][0] + ) mask_label = self._compute_mask(label_lengths) label_ids = tf_batch_data[LABEL_IDS][0] From 629ddce5b08f77030fb64fbcc354137fd04c034a Mon Sep 17 00:00:00 2001 From: Tanja Bergmann Date: Tue, 14 Apr 2020 17:08:35 +0200 Subject: [PATCH 16/18] move activation to transformer block --- rasa/nlu/classifiers/diet_classifier.py | 1 - rasa/utils/tensorflow/transformer.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 2d5414d9421b..558dd5e4218a 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -1246,7 +1246,6 @@ def _create_sequence( outputs = self._tf_layers[f"{name}_transformer"]( transformer_inputs, 1 - mask, self._training ) - outputs = tfa.activations.gelu(outputs) return outputs, inputs, seq_ids, lm_mask_bool diff --git a/rasa/utils/tensorflow/transformer.py b/rasa/utils/tensorflow/transformer.py index 51bfe3094508..4e35fafa0cc6 100644 --- a/rasa/utils/tensorflow/transformer.py +++ b/rasa/utils/tensorflow/transformer.py @@ -626,4 +626,7 @@ def call( # if normalization is done in encoding layers, then it should also be done # on the output, since the output can grow very large, being the sum of # a whole stack of unnormalized layer outputs. - return self._layer_norm(x) # (batch_size, length, units) + normalized_x = self._layer_norm(x) # (batch_size, length, units) + + # apply final activation + return tfa.activations.gelu(normalized_x) From 28c37f7a3b29ba782fccbc0ba05ae3373633ec68 Mon Sep 17 00:00:00 2001 From: Tanja Bergmann Date: Tue, 14 Apr 2020 17:11:16 +0200 Subject: [PATCH 17/18] add changelog --- changelog/5626.msic.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/5626.msic.rst diff --git a/changelog/5626.msic.rst b/changelog/5626.msic.rst new file mode 100644 index 000000000000..f7222708aa93 --- /dev/null +++ b/changelog/5626.msic.rst @@ -0,0 +1 @@ +Move ``tfa.activations.gelu(x)`` from ``DIETClassifier`` to transformer block. \ No newline at end of file From 7fffa52003b5da68c771d77208ae4f8f5ee86b95 Mon Sep 17 00:00:00 2001 From: Tanja Bergmann Date: Fri, 24 Apr 2020 11:35:19 +0200 Subject: [PATCH 18/18] reapplying changes --- CHANGELOG.rst | 4 ---- changelog/5626.msic.rst | 1 - changelog/5631.misc.rst | 1 + rasa/nlu/classifiers/diet_classifier.py | 10 +++++++--- rasa/utils/tensorflow/transformer.py | 5 +---- 5 files changed, 9 insertions(+), 12 deletions(-) delete mode 100644 changelog/5626.msic.rst create mode 100644 changelog/5631.misc.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9fac1be86c93..cc5b8da5b2f6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -36,10 +36,6 @@ Bugfixes Therefore, all 1.9 models `except for 1.9.6` will be compatible; a model trained on 1.9.6 will need to be retrained on 1.9.7. -Miscellaneous internal changes ------------------------------- -- #5631 - [1.9.6] - 2020-04-15 ^^^^^^^^^^^^^^^^^^^^ diff --git a/changelog/5626.msic.rst b/changelog/5626.msic.rst deleted file mode 100644 index f7222708aa93..000000000000 --- a/changelog/5626.msic.rst +++ /dev/null @@ -1 +0,0 @@ -Move ``tfa.activations.gelu(x)`` from ``DIETClassifier`` to transformer block. \ No newline at end of file diff --git a/changelog/5631.misc.rst b/changelog/5631.misc.rst new file mode 100644 index 000000000000..d1994407f85c --- /dev/null +++ b/changelog/5631.misc.rst @@ -0,0 +1 @@ +Set regularization constant in ``EmbeddingIntentClassifier`` to 0.001. \ No newline at end of file diff --git a/rasa/nlu/classifiers/diet_classifier.py b/rasa/nlu/classifiers/diet_classifier.py index 240d942b1062..32d6966627b4 100644 --- a/rasa/nlu/classifiers/diet_classifier.py +++ b/rasa/nlu/classifiers/diet_classifier.py @@ -1255,7 +1255,7 @@ def _create_sequence( seq_ids = None inputs = self._combine_sparse_dense_features( - features, mask, name, sparse_dropout, dense_dropout, + features, mask, name, sparse_dropout, dense_dropout ) inputs = self._tf_layers[f"ffnn.{name}"](inputs, self._training) @@ -1272,6 +1272,10 @@ def _create_sequence( transformer_inputs, 1 - mask, self._training ) + if self.config[NUM_TRANSFORMER_LAYERS] > 0: + # apply activation + outputs = tfa.activations.gelu(outputs) + return outputs, inputs, seq_ids, lm_mask_bool def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: @@ -1283,7 +1287,7 @@ def _create_all_labels(self) -> Tuple[tf.Tensor, tf.Tensor]: mask_label = self._compute_mask(label_lengths) x = self._create_bow( - self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name, + self.tf_label_data[LABEL_FEATURES], mask_label, self.label_name ) all_labels_embed = self._tf_layers[f"embed.{LABEL}"](x) @@ -1432,7 +1436,7 @@ def batch_loss( label_ids = tf_batch_data[LABEL_IDS][0] label = self._create_bow( - tf_batch_data[LABEL_FEATURES], mask_label, self.label_name, + tf_batch_data[LABEL_FEATURES], mask_label, self.label_name ) loss, acc = self._calculate_label_loss(cls, label, label_ids) self.intent_loss.update_state(loss) diff --git a/rasa/utils/tensorflow/transformer.py b/rasa/utils/tensorflow/transformer.py index 4e35fafa0cc6..51bfe3094508 100644 --- a/rasa/utils/tensorflow/transformer.py +++ b/rasa/utils/tensorflow/transformer.py @@ -626,7 +626,4 @@ def call( # if normalization is done in encoding layers, then it should also be done # on the output, since the output can grow very large, being the sum of # a whole stack of unnormalized layer outputs. - normalized_x = self._layer_norm(x) # (batch_size, length, units) - - # apply final activation - return tfa.activations.gelu(normalized_x) + return self._layer_norm(x) # (batch_size, length, units)