added hf token

.github/workflows/test-language-lang_id.yml

@@ -69,6 +69,10 @@ jobs:
                   echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
     test-src:
         runs-on: ubuntu-22.04
+        env:
+            HF_READ_ACCESS_TOKEN: ${{ secrets.HF_READ_ACCESS_TOKEN }}
+            DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+            DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
         steps:
             - name: Checkout
               uses: actions/checkout@v4

transforms/language/lang_id/test/test_lang_id.py

@@ -10,6 +10,7 @@
 # limitations under the License.
 ################################################################################
 
+import os
 import pyarrow as pa
 from data_processing.test_support.transform.table_transform_test import (
     AbstractTableTransformTest,
@@ -26,13 +27,14 @@ class TestLangIdentificationTransform(AbstractTableTransformTest):
 
     def get_test_transform_fixtures(self) -> list[tuple]:
         config = {
-            "model_credential": "PUT YOUR OWN HUGGINGFACE CREDENTIAL",
+            "model_credential": os.environ.get('HF_READ_ACCESS_TOKEN', "PUT YOUR OWN HUGGINGFACE CREDENTIAL"),
             "model_kind": KIND_FASTTEXT,
             "model_url": "facebook/fasttext-language-identification",
             "content_column_name": "contents",
             "output_lang_column_name": "l",
             "output_score_column_name": "s",
         }
+        print(os.environ)
         table = pa.Table.from_arrays(
             [
                 pa.array(

transforms/language/lang_id/test/test_lang_id_python.py

@@ -28,7 +28,7 @@ class TestPythonLangIdentificationTransform(AbstractTransformLauncherTest):
 
     def get_test_transform_fixtures(self) -> list[tuple]:
         cli_params = {
-            "lang_id_model_credential": "PUT YOUR OWN HUGGINGFACE CREDENTIAL",
+            "lang_id_model_credential": os.environ.get('HF_READ_ACCESS_TOKEN', "PUT YOUR OWN HUGGINGFACE CREDENTIAL"),
             "lang_id_model_kind": KIND_FASTTEXT,
             "lang_id_model_url": "facebook/fasttext-language-identification",
             "lang_id_content_column_name": "text",

transforms/language/lang_id/test/test_lang_id_ray.py

@@ -37,7 +37,7 @@ def get_test_transform_fixtures(self) -> list[tuple]:
         basedir = "../test-data"
         basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), basedir))
         config = {
-            model_credential_cli_param: "PUT YOUR OWN HUGGINGFACE CREDENTIAL",
+            model_credential_cli_param: os.environ.get('HF_READ_ACCESS_TOKEN', "PUT YOUR OWN HUGGINGFACE CREDENTIAL"),
             model_kind_cli_param: KIND_FASTTEXT,
             model_url_cli_param: "facebook/fasttext-language-identification",
             content_column_name_cli_param: "text",

transforms/language/lang_id/test/test_nlp.py

@@ -10,15 +10,19 @@
 # limitations under the License.
 ################################################################################
 
+import os
 import pyarrow as pa
 from dpk_lang_id.lang_models import KIND_FASTTEXT, LangModelFactory
 from dpk_lang_id.nlp import get_lang_ds_pa
 
 
 def test_language_identification():
     nlp_langid = LangModelFactory.create_model(
-        KIND_FASTTEXT, "facebook/fasttext-language-identification", "YOUR HUGGING FACE ACCOUNT TOKEN"
+        KIND_FASTTEXT, 
+        "facebook/fasttext-language-identification", 
+        os.environ.get('HF_READ_ACCESS_TOKEN', "YOUR HUGGING FACE ACCOUNT TOKEN")
     )
+
     documents = pa.array(
         [
             "Der Tell Sabi Abyad („Hügel des weißen Jungen“) ist eine historische "