initial

.gitignore

@@ -0,0 +1,63 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+# next.js build output
+.next
+
+out

app.js

@@ -0,0 +1,173 @@
+const fs=require('fs');
+const request=require('request');
+const iconv=require('iconv-lite');
+const urlModule = require('url');
+const querystring=require('querystring');
+const cliProgress = require('cli-progress');
+const nodeTar = require('tar');
+const PDFMerger = require('pdf-merger-js');
+var sameTime = new require('./sameTime.js');
+var downloadLimit=new sameTime(50);
+
+//Разрешить любые поддельные сертификаты, например от грибова
+process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = 0;
+process.env['NO_PROXY'] = 'www.caica.ru';
+
+var books=[
+  {
+    url:'http://www.caica.ru/common/AirInter/validaip/html/menurus.htm',
+    outfile:'AIPRUS1'
+  },
+  {
+    url:'http://www.caica.ru/common/AirClassABV/validaip2/html/menurus.htm',
+    outfile:'AIPRUS2'
+  },
+  {
+    url:'http://www.caica.ru/common/AirClassGDE/validaip4/html/menurus.htm',
+    outfile:'AIPRUS4'
+  }
+];
+
+
+var re=/\"([^"]+\.pdf)\",\"([^"]+)\"/;
+var bookFolders=[];
+function nextBook(){
+  var book=books.shift();
+  var files=[];
+
+  if (!book){
+    console.log('pack folders');
+    nodeTar.c({
+      gzip: false,//<true|gzip options>,
+      file: 'out/AIP.tar'
+    },bookFolders).then(_ => {
+      console.log('finish!');
+    });
+    return;
+  }
+  bookFolders.push('out/'+book.outfile);
+  //return nextBook();
+
+  var bar1 = new cliProgress.Bar({
+    format: book.outfile+' [{bar}] {percentage}% | ETA: {eta}s | {value}/{total}'
+  }, cliProgress.Presets.shades_classic);
+  request.get({url:book.url,encoding:null},(err,resp,menuBodyBuf)=>{
+    if (err){
+      console.log(err);
+      process.exit(10);
+    }
+
+    var menuBody=iconv.decode(menuBodyBuf,'win1251').split(/\n/);
+    var groups=[];
+    menuBody.forEach((line)=>{
+      if (!re.test(line)){
+        return;
+      }
+
+      var reResult=re.exec(line);
+      var fn=reResult[1];
+      var dirs=fn.split(/\//g);
+      dirs.pop();
+      //!/2\-ad2\-rus\-unne\-069\.pdf/.test(line) &&
+      //!/2\-sup\-2019\-03\_rus\.pdf/.test(line) &&
+      ///\s\.pdf$/.test(reResult[1]) &&
+      var group=dirs.pop();
+      var g1=dirs.pop()||'';
+      var g2=dirs.pop()||'';
+
+      if (g2=='ad2' && /[a-zA-Z]{4,4}/.test(group)){
+        group=group.replace(/([a-zA-Z]{2,2})([a-zA-Z]{2,2})/,'$1');
+        group='aerodrome_'+group;
+      }
+
+      group=group.replace(/^(enr)([0-9\-]+)$/,'$1');
+      group=group.replace(/^(gen)([0-9\-]+)$/,'$1');
+      group=group.replace(/^(u[a-zA-Z])([a-zA-Z]{2,2})$/,'$1');
+      //group=group.replace(/^(ad)([0-9\-]+)$/,'$1');
+      //group=group.replace(/^(aip|aic|amdt)$/,'begining');
+
+
+      groups.indexOf(group)==-1 && groups.push(group);
+
+      files.push({
+        group:group,
+        link:urlModule.resolve(book.url,fn),
+        name:reResult[2]
+      });
+    });
+
+    if (files.length==0){
+      console.log("Body:%s\nError, no files found",iconv.decode(menuBodyBuf,'win1251'));
+      process.exit(9);
+    }
+    //files.splice(0, Math.floor(files.length*3/4));
+
+    bar1.start(files.length, 0);
+
+
+    //files.splice(10);
+    var downloadedCnt=0;
+
+
+    function finishBook(){
+      bar1.stop();
+      var merger = new PDFMerger();
+      var group=files[0].group;
+
+      groups.forEach((group,groupInd)=>{
+        merger = new PDFMerger();
+        files.forEach((file)=>{
+          try{
+            if (file.group==group){
+              merger.add(file.pdfBuf);
+              file.pdfBuf=null;
+            }
+          }catch(e){
+            console.log('err',file.link,e);
+          }
+        });
+        fs.mkdirSync('out/'+book.outfile,{recursive:true})
+        merger.save('out/'+book.outfile+'/'+(groupInd+1)+'_'+book.outfile+'_'+group+'.pdf');
+      });
+
+      nextBook();
+    }
+
+    function getNextPdf(file){
+      function tryLoad(file){
+        request.get({
+          url:/[а-яА-Я]/.test(file.link)?encodeURI(file.link):file.link,
+          encoding:null,
+          timeout:60000
+        },(err,resp,pdfBuf)=>{
+          err && console.log('file',file.link,err,resp && resp.statusCode,pdfBuf && pdfBuf.length);
+          if (resp && resp.statusCode==200){
+            if (Number(resp.headers['content-length']!=pdfBuf.length)){
+              console.log('!!!WTF rather buffer size');
+              tryLoad(file);
+              return
+            }
+            file.pdfBuf=pdfBuf;
+            downloadedCnt++;
+            bar1.update(downloadedCnt);
+
+            (downloadedCnt==files.length) && finishBook();
+            //merger.add(pdfBuf);
+          }else{
+            tryLoad(file);
+            return;
+          }
+          downloadLimit.e();
+        });
+      }
+      downloadLimit.run(tryLoad,file);
+    }
+
+    files.forEach(getNextPdf);
+    //getNextPdf();
+    //urlModule.resolve(options.playlist,url);
+    //console.log(menuBody);
+
+  });
+}
+nextBook();