Doc download job

Author: BigBird01

data_creation/batch_job.sh

@@ -7,28 +7,27 @@ mkdir -p $TARGET_DIR
 rsync -ruC --exclude  pre_computed  --exclude processed_data --exclude *.pyc $SOURCE/ $TARGET_DIR
 
 work_dir=$TARGET_DIR/
-log_dir=/mount/biglm_data/ELI5/logs/
-output_dir=/mount/biglm_data/ELI5/processed_data
+log_dir=/mount/biglm_data/ELI5/logs_docs/
 mkdir -p $log_dir
 
-export WORLD_SIZE=9
+export WORLD_SIZE=16
 nodes=$(python -c "for i in range(1,$WORLD_SIZE):  print(i)")
 node_ids=($(python -c "for i in range(0,$WORLD_SIZE):  print(i)"))
-years=($(python -c "for i in range(2011,2011+$WORLD_SIZE):  print(i)"))
+#years=($(python -c "for i in range(2011,2011+$WORLD_SIZE):  print(i)"))
 #master=$(ssh -x -o LogLevel=ERROR worker-1 "echo \$hostname" )
 for i in $nodes; do
 	worker=worker-${node_ids[$i]}
 	sy=${years[$i]}
-	kill -9 $(ps -x |grep "download_reddit_qalist.py" |grep -v grep|awk -F' ' '{print $1}')
+	rank=$i
 	ssh -x -o LogLevel=ERROR  $worker "mkdir -p $TARGET_DIR"
 	rsync -ruC  --exclude  pre_computed  --exclude processed_data -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"  $TARGET_DIR/ $worker:$TARGET_DIR
-	ssh -x -o LogLevel=ERROR  $worker "mkdir -p $log_dir; cd $work_dir; nohup ./download_qa.sh $sy > ${log_dir}/nohup_${i}.std  2> ${log_dir}/nohup_${i}.err &"
+	ssh -x -o LogLevel=ERROR  $worker "mkdir -p $log_dir; cd $work_dir; nohup ./download_docs.sh $rank > ${log_dir}/nohup_${i}.std  2> ${log_dir}/nohup_${i}.err &"
 done
 
 i=0
+rank=$i
 worker=worker-${node_ids[$i]}
 sy=${years[$i]}
-kill -9 $(ps -x |grep "download_reddit_qalist.py" |grep -v grep|awk -F' ' '{print $1}')
 ssh -x -o LogLevel=ERROR  $worker "mkdir -p $TARGET_DIR"
 rsync -ruC  --exclude  pre_computed  --exclude processed_data -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"  $TARGET_DIR/ $worker:$TARGET_DIR
-ssh -x -o LogLevel=ERROR  $worker "mkdir -p $log_dir; cd $work_dir; ./download_qa.sh $sy"
+ssh -x -o LogLevel=ERROR  $worker "mkdir -p $log_dir; cd $work_dir; ./download_docs.sh $rank"

data_creation/download_docs.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+SCRIPT=$(readlink -f "$0")
+SCRIPT_DIR=$(dirname "$SCRIPT")
+cd $SCRIPT_DIR
+rank=$1
+
+#pip install -r requirements.txt
+output=/mount/biglm_data/ELI5/processed_data/support_docs_$rank
+pre_computed=/mount/biglm_data/ELI5/pre_computed
+mkdir -p $output
+slsize=$[71520/16]
+
+kill -9 $(ps -x |grep "download_support_docs.py" |grep -v grep|awk -F' ' '{print $1}')
+python download_support_docs.py  --slnum $rank --slsize $slsize --subreddit_names '["explainlikeimfive", "AskHistorians", "askscience"]' --output_dir $output \
+	--wet_urls ${pre_computed}/wet.paths --pre_computed_dir ${pre_computed}

data_creation/download_qa.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 SCRIPT=$(readlink -f "$0")
 SCRIPT_DIR=$(dirname "$SCRIPT")
-cd SCRIPT_DIR
+cd $SCRIPT_DIR
 year=$1
 
 pip install -r requirements.txt

data_creation/download_support_docs.py

@@ -29,6 +29,8 @@ def main():
                         help='how often are results written to file')
     parser.add_argument('-o', '--output_dir', default='processed_data/collected_docs', type=str,
                         help='where to save the output')
+    parser.add_argument('--pre_computed_dir', default='pre_computed', type=str,
+                        help='where to load pre_computed')
     args    = parser.parse_args()
     # parse full list of wet urls
     # slice urls for WET files can be found at https://commoncrawl.org/2018/08/august-2018-crawl-archive-now-available/
@@ -43,7 +45,7 @@ def main():
     sr_names        = json.loads(args.subreddit_names)
     for name in sr_names:
         print(name)
-        ccrawl_ids_maps[name]   = json.load(open('pre_computed/%s_ccrawl_ids.json' % (name,)))
+        ccrawl_ids_maps[name]   = json.load(open(f'{args.pre_computed_dir}/{name}_ccrawl_ids.json'))
         for i, (k, _) in enumerate(ccrawl_ids_maps[name]):
             reddit_id_group[k]  = (i * 10) // len(ccrawl_ids_maps[name])
     # make a list of the CommonCrawl UIDs we want to process and keep