change category separator

from "_" to "-"
2DegreesInvesting · Apr 19, 2024 · d28e255 · d28e255
1 parent d7fed91
commit d28e255
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/src/main.py b/src/main.py
@@ -10,7 +10,7 @@ async def main():
     if 'DATABRICKS_RUNTIME_VERSION' in os.environ:
         dbutils.library.restartPython()
     # Execute EuroPagesScraper
-    await EuroPagesProductsScraper().scrape_and_export("company_scraper", "italy", "agriculture_livestock", "agricultural_production")
+    await EuroPagesProductsScraper().scrape_and_export("company_scraper", "italy", "agriculture-livestock", "agricultural-production")
 
 # Run scraper
 if __name__ == "__main__":

diff --git a/src/utils.py b/src/utils.py
@@ -112,9 +112,9 @@ async def extract_ep_subsectors(self, url, session):
                 sector = " ".join(soup.find("div", class_="v-breadcrumbs__item").text.strip().split())
 
                 cats_sectors_and_subsectors.append([self.generate_hash_id(subsector.text.strip()), 
-                                                    re.sub('[^0-9a-zA-Z]+', "_", category.lower()), 
-                                                    re.sub('[^0-9a-zA-Z]+', "_", sector.lower()), 
-                                                    re.sub('[^0-9a-zA-Z]+', "_", subsector.text.strip().lower()), 
+                                                    re.sub('[^0-9a-zA-Z]+', "-", category.lower()), 
+                                                    re.sub('[^0-9a-zA-Z]+', "-", sector.lower()), 
+                                                    re.sub('[^0-9a-zA-Z]+', "-", subsector.text.strip().lower()), 
                                                     "https://www.europages.co.uk/companies/{}.html".format(quote(sector.lower())),
                                                     "https://www.europages.co.uk{}".format(subsector["href"])])
             return cats_sectors_and_subsectors
@@ -257,7 +257,7 @@ async def scrape_and_export(self, type="product_updater", country=None, group=No
             categorization = pd.read_csv(input)
             subsector_urls = categorization[categorization["sector"] == sector]["subsector_url"].tolist()
             sector_url = categorization[categorization["sector"] == sector]["sector_url"].values[0]
-            out = f"{country}-{group}-{sector}.csv"
+            out = f"{country}_{group}_{sector}.csv"
 
             # store all company_information
             all_company_info = []