return json_file_string
def load_input_files(dir=CLEANED_DATA_DIR):
directory = os.fsencode(dir)
file_string = ""
for file in os.listdir(directory):
filename = os.fsdecode(file)
if filename.endswith(".txt"):
file_string += load_text_file(filename, dir)
elif filename.endswith(".csv"):
file_string += load_csv_file(filename, dir)
elif filename.endswith(".json"):
file_string += load_json_file(filename, "text", dir)
return file_string
# TODO: Refactor to take a string input
# This should just take a string input and the data load calls should be
# handled in the chatbot module
# labels: refactor
# assignees: danmassarano
def train_text_model(dir=CLEANED_DATA_DIR):
print("Loading data...", end="")
file_string = load_input_files(dir)
print(" done")
print("Training model...", end="")
This should just take a string input and the data load calls should be
handled in the chatbot module
https://github.com/danmassarano/chatbot/blob/23534ddfc1105813c93bc12a84e81860ee76b275/src/markov_chain.py#L48