Merge pull request #36 from TailUFPB/develop

Mudar a forma de criar o executavel para que seja mais rapido

.github/workflows/develop.yml

@@ -1,69 +1,30 @@
 name: CI
-# Controls when the workflow will run
 on:
  push:
  branches: ["develop"]
 
-# Prevent duplicate workflows from running
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
 
 jobs:
- # Static tests don't involve any logic or context.
- # They are just a set of tests that can detect if we are not introducing any faulty code.
- static-test:
- name: 🔬 Static tests
- runs-on: ubuntu-latest
- steps:
- - name: ⬇️ Checkout repo
- uses: actions/checkout@v4
- with:
- fetch-depth: 2
-
- - name: ⎔ Setup Node
- uses: actions/setup-node@v3
- with:
- cache: "npm"
-
- - name: 🟨 Setup Python
- uses: actions/setup-python@v3
-
- - name: 📦 Install Node dependencies
- run: npm install
-
- # # Unit tests are tests that are not dependent on any external service.
- # # Usually, they are tests that are testing the logic of a specific function or component.
- unit-test:
- needs: [static-test]
- name: 🚦 Unit tests
- runs-on: ubuntu-latest
+ build:
+ name: 🏗️ Build
+ runs-on: windows-latest
+ strategy:
+ fail-fast: false
  steps:
  - name: ⬇️ Checkout repo
  uses: actions/checkout@v4
  with:
  fetch-depth: 2
 
- - name: ⎔ Setup Node
- uses: actions/setup-node@v3
+ - name: Set up conda
+ uses: conda-incubator/setup-miniconda@v2
  with:
- cache: "npm"
-
- - name: 🟨 Setup Python
- uses: actions/setup-python@v3
-
- - name: 📦 Install dependencies
- run: npm install
+ activate-environment: default
+ environment-file: api/environment.yml
 
- - name: 🚦 Run unit tests
- run: npm test
-
- # # Integration tests are tests that are dependent on external services.
- integration-test:
- needs: [static-test]
- name: 🚥 Integration tests
- runs-on: ubuntu-latest
- steps:
  - name: ⬇️ Checkout repo
  uses: actions/checkout@v4
  with:
@@ -76,65 +37,26 @@ jobs:
 
  - name: 🟨 Setup Python
  uses: actions/setup-python@v3
-
- - name: 📦 Install dependencies
- run: npm install
-
- # - name: 🐳 Docker compose
- # run:
- # docker-compose up -d && sleep 3 && pnpm prisma migrate reset --force
- # --skip-seed
-
- - name: 🚦 Run integration tests
- run: npm test
-
- # Create Build
- build:
- needs: [static-test, unit-test, integration-test]
- name: 🏗️ Build
- runs-on: ubuntu-latest
- steps:
- - name: Set up Wine
- uses: actions/setup-node@v2
- with:
- node-version: "14"
-
- - name: Install Wine64
- run: sudo apt update && sudo apt install wine64
-
- - name: Install Wine32
- run: |
- sudo dpkg --add-architecture i386
- sudo apt-get update
- sudo apt-get install wine32
-
- - name: ⬇️ Checkout repo
- uses: actions/checkout@v4
  with:
- fetch-depth: 2
+ python-version: 3.12.3
 
- - name: ⎔ Setup Node
- uses: actions/setup-node@v3
- with:
- cache: "npm"
+ - name: Install pip dependencies
+ run: pip install -r api/requirements.txt pyinstaller==6.6.0
 
- - name: 🟨 Setup Python
- uses: actions/setup-python@v3
+ - name: Build flask exe
+ run: pyinstaller api/app.spec
 
- - name: 📦 Install dependencies
+ - name: Install dependencies
  run: npm install
 
- - name: 🏗️ Build
- run: npm run build
-
- #- name: 📦 Electron Package
- # run: npx electron-packager . LinguifAI --platform=win32 --arch=x64 --out=dist
-
  - name: 📦 Electron Builder
  run: npm run electron:package:win
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+ - name: Print directory tree
+ run: tree
+
  - name: Get latest release number
  id: get_latest_release
  uses: actions/github-script@v4
@@ -154,7 +76,7 @@ jobs:
  console.log(`::set-output name=new_version::${newVersion}`);
 
  - name: Rename file
- run: mv -f dist/LinguifAI\ Setup\ 0.1.0.exe dist/LinguifAI\ Setup\ ${{ steps.get_latest_release.outputs.new_version }}.exe
+ run: ren "dist\LinguifAI Setup 0.1.0.exe" "LinguifAI Setup ${{ steps.get_latest_release.outputs.new_version }}.exe"
 
  - name: Create Release
  id: create_release
@@ -165,5 +87,4 @@ jobs:
  tag_name: v${{ steps.get_latest_release.outputs.new_version }}
  name: v${{ steps.get_latest_release.outputs.new_version }}
  prerelease: true
- body: |
- Descrição do release aqui
+ body: ${{ github.event.head_commit.message }}

.github/workflows/main.yml

@@ -92,21 +92,20 @@ jobs:
  build:
  needs: [static-test, unit-test, integration-test]
  name: 🏗️ Build
- runs-on: ubuntu-latest
+ runs-on: windows-latest
+ strategy:
+ fail-fast: false
  steps:
- - name: Set up Wine
- uses: actions/setup-node@v2
+ - name: ⬇️ Checkout repo
+ uses: actions/checkout@v4
  with:
- node-version: "14"
-
- - name: Install Wine64
- run: sudo apt update && sudo apt install wine64
+ fetch-depth: 2
 
- - name: Install Wine32
- run: |
-  sudo dpkg --add-architecture i386
- sudo apt-get update
- sudo apt-get install wine32
+ - name: Set up conda
+ uses: conda-incubator/setup-miniconda@v2
+ with:
+ activate-environment: default
+ environment-file: api/environment.yml
 
  - name: ⬇️ Checkout repo
  uses: actions/checkout@v4
@@ -120,21 +119,26 @@ jobs:
 
  - name: 🟨 Setup Python
  uses: actions/setup-python@v3
+ with:
+ python-version: 3.12.3
 
- - name: 📦 Install dependencies
- run: npm install
+ - name: Install pip dependencies
+ run: pip install -r api/requirements.txt pyinstaller==6.6.0
 
- - name: 🏗️ Build
- run: npm run build
+ - name: Build flask exe
+ run: pyinstaller api/app.spec
 
- #- name: 📦 Electron Package
- # run: npx electron-packager . LinguifAI --platform=win32 --arch=x64 --out=dist
+ - name: Install dependencies
+ run: npm install
 
  - name: 📦 Electron Builder
  run: npm run electron:package:win
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+ - name: Print directory tree
+ run: tree
+
  - name: Get latest release number
  id: get_latest_release
  uses: actions/github-script@v4
@@ -145,7 +149,7 @@ jobs:
  owner: context.repo.owner,
  repo: context.repo.repo,
  per_page: 10
- }); 
+ });
 
  const latestPreRelease = response.data[0];
  const preReleaseTag = latestPreRelease.name;
@@ -154,8 +158,21 @@ jobs:
  console.log(`::set-output name=new_version::${newVersion}`);
 
  - name: Rename file
- run: mv -f dist/LinguifAI\ Setup\ 0.1.0.exe dist/LinguifAI\ Setup\ ${{ steps.get_latest_release.outputs.new_version }}.exe
+ run: ren "dist\LinguifAI Setup 0.1.0.exe" "LinguifAI Setup ${{ steps.get_latest_release.outputs.new_version }}.exe"
 
+ - name: Get latest commit message
+ id: get_latest_commit_message
+ uses: actions/github-script@v4
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const response = await github.repos.listCommits({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ per_page: 1,
+ });
+ console.log(`::set-output name=commit_message::${response.data[0].commit.message}`);
+ 
  - name: Create Release
  id: create_release
  uses: softprops/action-gh-release@v1
@@ -165,5 +182,4 @@ jobs:
  tag_name: v${{ steps.get_latest_release.outputs.new_version }}
  name: v${{ steps.get_latest_release.outputs.new_version }}
  prerelease: true
- body: |
- Descrição do release aqui
+ body: ${{ steps.get_latest_commit_message.outputs.commit_message }}

api/DataProcesser.py

@@ -2,13 +2,18 @@
 from NbNewsModel import news_prediction
 from NbEmotionsModel import make_prediction
 from available_classifiers import get_available_classifiers
+from sklearn.pipeline import make_pipeline
+
+# bag of words
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+from sklearn.model_selection import train_test_split
+from sklearn.naive_bayes import MultinomialNB
 
 import Neural_Network2
-import pickle
 import re
 import joblib
 import numpy as np
-import string
 import os
 import pandas as pd
 import torch
@@ -30,7 +35,8 @@ def handle_classify(self, df, classifier):
  classifier_switcher = get_available_classifiers() # id: nome_arquivo
  model_name = classifier_switcher[classifier]
  if model_name.endswith('.pkl'):
- return self.pretrained_predict(df, model_name)
+ pipeline = self.get_pipeline(model_name)
+ return self.pretrained_predict(df, pipeline)
  else:
  return self.trained_predict(df, model_name)
  #classifier_switcher = {
@@ -41,6 +47,20 @@ def handle_classify(self, df, classifier):
 
  #return classifier_switcher.get(classifier, lambda: "Invalid Classifier")(df)
 
+ def get_pipeline(self, model_name):
+ if model_name=="emotion_pipeline.pkl":
+ df = pd.read_csv('api/training_df/tweet_emotions.csv')
+ train_data, test_data, train_target, test_target = train_test_split(df['content'], df['sentiment'], test_size=0.2, shuffle=True)
+ elif model_name=="hate_speech.pkl":
+ df = pd.read_csv('api/training_df/nb_hatespeech.csv', sep=';')
+ train_data, test_data, train_target, test_target = train_test_split(df['comment'], df['isHate'], test_size=0.2, shuffle=True)
+ elif model_name=="text_classification_pipeline.pkl":
+ df = pd.read_csv('api/training_df/nb_news.csv')
+ train_data, test_data, train_target, test_target = train_test_split(df['short_description'], df['category'], test_size=0.2, shuffle=True)
+ else:
+ return None
+ return make_pipeline(TfidfVectorizer(), MultinomialNB()).fit(train_data, train_target)
+
  def generate_statistics(self, df):
  unique_labels = df['output_column'].unique()
 
@@ -67,19 +87,15 @@ def classify_emotions(self, df):
  df['output_column'] = df['input_column'].apply(make_prediction)
  return df
 
- def lin_regression_model(self, df):
- df['output_column'] = df['input_column'].apply(make_prediction_nblin)
- return df
+ # def lin_regression_model(self, df):
+ #  df['output_column'] = df['input_column'].apply(make_prediction_nblin)
+ #  return df
 
  def nb_news_application(self, df):
  df['output_column'] = df['input_column'].apply(news_prediction)
  return df
 
- def pretrained_predict(self, df, model_name):
- model_file = f'api/models/{model_name}'
- with open(model_file, 'rb') as model:
- pipeline = pickle.load(model)
-
+ def pretrained_predict(self, df, pipeline):
  texts_to_predict = df['input_column']
  texts_to_predict = [str(text) for text in texts_to_predict]
  predictions = pipeline.predict(texts_to_predict)