fetch and extract course data from raw url

Author: Yukaii

.travis/deploy.sh

@@ -18,7 +18,20 @@ fi
 
 if [[ "$TRAVIS_PULL_REQUEST" != "false" || "$TRAVIS_BRANCH" == "$SOURCE_BRANCH" ]]; then
   echo 'start running crawler'
-  ./bin/ntust
+  REPO_BASE_URL=`git config remote.origin.url | sed 's/\.git//g'`
+  while IFS= read -r org_code
+  do
+    RAW_DATA_URL="$REPO_BASE_URL/raw/data/$org_code.gz"
+
+    echo "downloading $org_code data from $RAW_DATA_URL"
+    curl -L $RAW_DATA_URL --output "$org_code.gz"
+
+    echo "unzipping..."
+    gzip -d < "$org_code.gz" > "$org_code.json"
+
+    echo "extracting..."
+    ./.travis/extract_json "$org_code.json"
+  done < "support_organizations.txt"
 fi
 
 # Save some useful information

support_organizations.txt

@@ -0,0 +1 @@
+ntust