Skip to content

Commit

Permalink
Merge pull request #5455 from RasaHQ/fix_training
Browse files Browse the repository at this point in the history 
sort nlu/story files before training
  • Loading branch information
@akelad
akelad authored Mar 23, 2020
2 parents b53024d + 5537ebc commit b6c837d
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 31 deletions.
2 changes: 2 additions & 0 deletions changelog/5405.bugfix.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fixes issue where model always gets retrained if multiple NLU/story files are in a
directory, by sorting the list of files.
8 changes: 4 additions & 4 deletions rasa/data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import tempfile
import uuid
import re
from typing import Tuple, List, Text, Set, Union, Optional
from typing import Tuple, List, Text, Set, Union, Optional, Iterable
from rasa.nlu.training_data import loading
from rasa.utils.io import DEFAULT_ENCODING

Expand Down Expand Up @@ -60,7 +60,7 @@ def get_core_nlu_directories(

def get_core_nlu_files(
paths: Optional[Union[Text, List[Text]]]
) -> Tuple[Set[Text], Set[Text]]:
) -> Tuple[List[Text], List[Text]]:
"""Recursively collects all training files from a list of paths.
Args:
Expand Down Expand Up @@ -95,7 +95,7 @@ def get_core_nlu_files(
story_files.update(new_story_files)
nlu_data_files.update(new_nlu_data_files)

return story_files, nlu_data_files
return sorted(story_files), sorted(nlu_data_files)


def _find_core_nlu_files_in_directory(directory: Text,) -> Tuple[Set[Text], Set[Text]]:
Expand Down Expand Up @@ -203,7 +203,7 @@ def is_config_file(file_path: Text) -> bool:
return file_name in ["config.yml", "config.yaml"]


def _copy_files_to_new_dir(files: Set[Text]) -> Text:
def _copy_files_to_new_dir(files: Iterable[Text]) -> Text:
directory = tempfile.mkdtemp()
for f in files:
# makes sure files do not overwrite each other, hence the prefix
Expand Down
4 changes: 2 additions & 2 deletions rasa/importers/multi_project.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ def __init__(
extra_story_files, extra_nlu_files = data.get_core_nlu_files(
training_data_paths
)
self._story_paths += list(extra_story_files)
self._nlu_paths += list(extra_nlu_files)
self._story_paths += extra_story_files
self._nlu_paths += extra_nlu_files

logger.debug(
"Selected projects: {}".format("".join([f"\n-{i}" for i in self._imports]))
Expand Down
50 changes: 25 additions & 25 deletions tests/core/test_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,48 +110,48 @@ def test_same_file_names_get_resolved(tmpdir):
[
(
"dialogflow",
{
[
"data/examples/dialogflow/agent.json",
"data/examples/dialogflow/entities/cuisine.json",
"data/examples/dialogflow/intents/affirm.json",
"data/examples/dialogflow/entities/location_entries_es.json",
"data/examples/dialogflow/intents/affirm_usersays_en.json",
"data/examples/dialogflow/intents/hi_usersays_es.json",
"data/examples/dialogflow/entities/cuisine_entries_en.json",
"data/examples/dialogflow/entities/cuisine_entries_es.json",
"data/examples/dialogflow/intents/inform_usersays_en.json",
"data/examples/dialogflow/intents/hi.json",
"data/examples/dialogflow/intents/goodbye_usersays_en.json",
"data/examples/dialogflow/agent.json",
"data/examples/dialogflow/intents/hi_usersays_en.json",
"data/examples/dialogflow/entities/location.json",
"data/examples/dialogflow/intents/affirm_usersays_es.json",
"data/examples/dialogflow/entities/cuisine_entries_en.json",
"data/examples/dialogflow/package.json",
"data/examples/dialogflow/entities/location_entries_en.json",
"data/examples/dialogflow/entities/location_entries_es.json",
"data/examples/dialogflow/intents/Default Fallback Intent.json",
"data/examples/dialogflow/intents/goodbye_usersays_es.json",
"data/examples/dialogflow/intents/affirm.json",
"data/examples/dialogflow/intents/affirm_usersays_en.json",
"data/examples/dialogflow/intents/affirm_usersays_es.json",
"data/examples/dialogflow/intents/goodbye.json",
"data/examples/dialogflow/entities/location_entries_en.json",
"data/examples/dialogflow/intents/goodbye_usersays_en.json",
"data/examples/dialogflow/intents/goodbye_usersays_es.json",
"data/examples/dialogflow/intents/hi.json",
"data/examples/dialogflow/intents/hi_usersays_en.json",
"data/examples/dialogflow/intents/hi_usersays_es.json",
"data/examples/dialogflow/intents/inform.json",
"data/examples/dialogflow/intents/inform_usersays_en.json",
"data/examples/dialogflow/intents/inform_usersays_es.json",
},
"data/examples/dialogflow/package.json",
],
),
(
"luis",
{
"data/examples/luis/demo-restaurants_v5.json",
"data/examples/luis/demo-restaurants_v4.json",
[
"data/examples/luis/demo-restaurants_v2.json",
},
"data/examples/luis/demo-restaurants_v4.json",
"data/examples/luis/demo-restaurants_v5.json",
],
),
(
"rasa",
{
[
"data/examples/rasa/demo-rasa-multi-intent.md",
"data/examples/rasa/demo-rasa-responses.md",
"data/examples/rasa/demo-rasa.json",
"data/examples/rasa/demo-rasa.md",
"data/examples/rasa/demo-rasa-responses.md",
"data/examples/rasa/demo-rasa-multi-intent.md",
},
],
),
("wit", {"data/examples/wit/demo-flights.json"}),
("wit", ["data/examples/wit/demo-flights.json"]),
],
)
def test_find_nlu_files_with_different_formats(test_input, expected):
Expand Down

0 comments on commit b6c837d

Please sign in to comment.