Merge branch 'main' of github.com:TailUFPB/LinguifAI

api/DataProcesser.py

@@ -1,3 +1,4 @@
+
 from NbNewsModel import news_prediction
 from NbEmotionsModel import make_prediction
 from NbLinRegressionModel import make_prediction_nblin
@@ -11,6 +12,7 @@
 import re
 import joblib
 import string
+import os
 
 import nltk
 from nltk.corpus import stopwords
@@ -27,7 +29,7 @@ def handle_classify(self, df, classifier):
  model_name = classifier_switcher[classifier]
  if model_name.endswith('.pkl'):
  return self.pretrained_predict(df, model_name)
- elif model_name.endswith('.keras'):
+ elif model_name.endswith('.h5'):
  return self.trained_predict(df, model_name)
  #classifier_switcher = {
  # 0: self.classify_emotions,
@@ -49,7 +51,7 @@ def generate_statistics(self, df):
  return statistics
 
  def preprocess_text(self, text):
- text = text.lower()
+ text = str(text).lower()
  text = re.sub('\[.*?\]', '', text)
  text = re.sub("\\W", " ", text)
  text = re.sub('https?://\S+|www\.\S+', '', text)
@@ -82,9 +84,19 @@ def pretrained_predict(self, df, model_name):
  df['output_column'] = predictions
  return df
 
+ def load_weights_and_model(self, name):
+ model_filename = f"api/models/{name}"
+ num_classes = model_filename[model_filename.index("/") + 1, model_filename.index("-")]
+ model = tf.keras.Sequential([
+ tf.keras.layers.Embedding(input_dim=20000, output_dim=128),
+ tf.keras.layers.LSTM(64),
+ tf.keras.layers.Dense(int(num_classes), activation='softmax')
+ ])
+ model.load_weights(model_filename)
+ return model
+
  def trained_predict(self, df, model_name):
- model_file = f'api/models/{model_name}'
- model = load_model(model_file)
+ model = self.load_weights_and_model(model_name)
 
  encoder_re = r'Trained-Model-(.*?).keras'
  encoder_name = re.search(encoder_re, model_name).group(1)
@@ -102,4 +114,7 @@ def trained_predict(self, df, model_name):
  df['output_column'] = predicted_labels
 
  return df
+
+
  ##TODO métodos com o processamento de classificação
+

api/Neural_Network2.py

@@ -17,7 +17,7 @@
 dirname = os.path.dirname(__file__)
 
 def preprocess_text(text):
- text = text.lower()
+ text = str(text).lower()
  text = re.sub('\[.*?\]', '', text)
  text = re.sub("\\W", " ", text)
  text = re.sub('https?://\S+|www\.\S+', '', text)
@@ -84,6 +84,8 @@ def create_and_train_model(train_texts, train_labels, name, epochs=5, batch_size
  joblib.dump(label_encoder, label_mapping_file)
 
  tfidf_vectorizer = TfidfVectorizer(max_features=20000)
+
+ train_texts = [preprocess_text(text) for text in train_texts]
  train_texts_tfidf = tfidf_vectorizer.fit_transform(train_texts)
 
  # Cria um conjunto de dados de texto usando a API de conjuntos de dados do TensorFlow

api/app.py

@@ -97,13 +97,17 @@ def train_model():
 def get_training_status():
  try:
  with open('training_progress.json', 'r') as file:
- data = json.load(file)
+ try:
+ data = json.load(file)
+ except json.decoder.JSONDecodeError:
+ return jsonify({'training_in_progress': True, 'training_progress': 0})
  training_status = data.get('training_in_progress', False)
  progress = data.get('training_progress', 0)
  return jsonify({'training_in_progress': training_status, 'training_progress': progress})
  except FileNotFoundError:
  return jsonify({'training_in_progress': False, 'training_progress': 0})
 
+
 #@app.teardown_appcontext
 #def teardown_appcontext(error=None):
  #shutdown_server()

package.json

@@ -30,7 +30,9 @@
  "typescript": "^4.9.5"
  },
  "scripts": {
- "start": "react-scripts start",
+ "start": "concurrently \"npm run start:react\" \"npm run start:flask\"",
+ "start:react": "react-scripts start",
+ "start:flask": "python api/app.py",
  "build": "react-scripts build",
  "test": "react-scripts test",
  "eject": "react-scripts eject",
@@ -55,6 +57,7 @@
  ]
  },
  "devDependencies": {
+ "concurrently": "^8.2.2",
  "electron": "^26.1.0",
  "tailwindcss": "^3.3.3"
  }

src/pages/train.tsx

@@ -46,15 +46,47 @@ export default function Train() {
 
  console.log(sendData);
 
- const response = await axios
- .post("http://localhost:5000/neural-network", sendData)
- .catch((error) => {
- console.error(error.response.data);
- });
 
- if (response && response.data) {
- console.log(response.data);
+ const maxRetries = 3;
+ let retryCount = 0;
+
+
+ const url = "http://localhost:5000/neural-network";
+
+
+ async function postData(url: string, data: { data: any[]; label: any[]; batch_size: number; epochs: number; learning_rate: number; name: string; }) {
+ try {
+ const response = await axios.post(url, data);
+ } catch (error) {
+ if (retryCount < maxRetries) {
+ retryCount++;
+ console.error(`Error occurred, retrying (attempt ${retryCount})...`);
+ postData(url, data); // Retry recursively
+ } else {
+ console.error("Max retry limit reached. Unable to post data.");
+ throw error; // Throw the error after maximum retries
+ }
+ }
  }
+
+ await axios
+ .post("http://localhost:5000/neural-network", sendData)
+ .catch(async (error) => {
+ await axios
+ .post("http://localhost:5000/neural-network", sendData)
+ .catch(async (error) => {
+ await axios
+ .post("http://localhost:5000/neural-network", sendData)
+ .catch(async (error) => {
+ await axios
+ .post("http://localhost:5000/neural-network", sendData)
+ .catch((error) => {
+ console.error(error.response.data);
+ })
+ })
+ })
+ });
+
  setIsLoading(false);
  };