diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 3108cca538..180c20e365 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -69,6 +69,10 @@ jobs: echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: runs-on: ubuntu-22.04 + env: + HF_READ_ACCESS_TOKEN: ${{ secrets.HF_READ_ACCESS_TOKEN }} + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} steps: - name: Checkout uses: actions/checkout@v4 diff --git a/transforms/language/lang_id/test/test_lang_id.py b/transforms/language/lang_id/test/test_lang_id.py index 10984d4a40..3efb489091 100644 --- a/transforms/language/lang_id/test/test_lang_id.py +++ b/transforms/language/lang_id/test/test_lang_id.py @@ -10,6 +10,7 @@ # limitations under the License. ################################################################################ +import os import pyarrow as pa from data_processing.test_support.transform.table_transform_test import ( AbstractTableTransformTest, @@ -26,13 +27,14 @@ class TestLangIdentificationTransform(AbstractTableTransformTest): def get_test_transform_fixtures(self) -> list[tuple]: config = { - "model_credential": "PUT YOUR OWN HUGGINGFACE CREDENTIAL", + "model_credential": os.environ.get('HF_READ_ACCESS_TOKEN', "PUT YOUR OWN HUGGINGFACE CREDENTIAL"), "model_kind": KIND_FASTTEXT, "model_url": "facebook/fasttext-language-identification", "content_column_name": "contents", "output_lang_column_name": "l", "output_score_column_name": "s", } + print(os.environ) table = pa.Table.from_arrays( [ pa.array( diff --git a/transforms/language/lang_id/test/test_lang_id_python.py b/transforms/language/lang_id/test/test_lang_id_python.py index 25e6702536..ca821c9f18 100644 --- a/transforms/language/lang_id/test/test_lang_id_python.py +++ b/transforms/language/lang_id/test/test_lang_id_python.py @@ -28,7 +28,7 @@ class TestPythonLangIdentificationTransform(AbstractTransformLauncherTest): def get_test_transform_fixtures(self) -> list[tuple]: cli_params = { - "lang_id_model_credential": "PUT YOUR OWN HUGGINGFACE CREDENTIAL", + "lang_id_model_credential": os.environ.get('HF_READ_ACCESS_TOKEN', "PUT YOUR OWN HUGGINGFACE CREDENTIAL"), "lang_id_model_kind": KIND_FASTTEXT, "lang_id_model_url": "facebook/fasttext-language-identification", "lang_id_content_column_name": "text", diff --git a/transforms/language/lang_id/test/test_lang_id_ray.py b/transforms/language/lang_id/test/test_lang_id_ray.py index 466db2d86b..23cffe5556 100644 --- a/transforms/language/lang_id/test/test_lang_id_ray.py +++ b/transforms/language/lang_id/test/test_lang_id_ray.py @@ -37,7 +37,7 @@ def get_test_transform_fixtures(self) -> list[tuple]: basedir = "../test-data" basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), basedir)) config = { - model_credential_cli_param: "PUT YOUR OWN HUGGINGFACE CREDENTIAL", + model_credential_cli_param: os.environ.get('HF_READ_ACCESS_TOKEN', "PUT YOUR OWN HUGGINGFACE CREDENTIAL"), model_kind_cli_param: KIND_FASTTEXT, model_url_cli_param: "facebook/fasttext-language-identification", content_column_name_cli_param: "text", diff --git a/transforms/language/lang_id/test/test_nlp.py b/transforms/language/lang_id/test/test_nlp.py index e61b2cc1cf..f6be51d707 100644 --- a/transforms/language/lang_id/test/test_nlp.py +++ b/transforms/language/lang_id/test/test_nlp.py @@ -10,6 +10,7 @@ # limitations under the License. ################################################################################ +import os import pyarrow as pa from dpk_lang_id.lang_models import KIND_FASTTEXT, LangModelFactory from dpk_lang_id.nlp import get_lang_ds_pa @@ -17,8 +18,11 @@ def test_language_identification(): nlp_langid = LangModelFactory.create_model( - KIND_FASTTEXT, "facebook/fasttext-language-identification", "YOUR HUGGING FACE ACCOUNT TOKEN" + KIND_FASTTEXT, + "facebook/fasttext-language-identification", + os.environ.get('HF_READ_ACCESS_TOKEN', "YOUR HUGGING FACE ACCOUNT TOKEN") ) + documents = pa.array( [ "Der Tell Sabi Abyad („Hügel des weißen Jungen“) ist eine historische "