init

Author: suqingdong

.gitignore

@@ -0,0 +1,3 @@
+*pyc
+*__pycache__
+.vscode

LICENSE

@@ -0,0 +1,3 @@
+include requirements.txt
+include nsfc/version/version.json
+include nsfc/data/*.db

MANIFEST.in

@@ -0,0 +1,12 @@
+# [LetPub](http://www.letpub.com.cn/index.php?page=grant)
+> 查询快，但频率会有限制
+> 目前数据只更新到了2019年，2020年数据和官网一样受限
+
+
+# [MedSci](https://www.medsci.cn/sci/nsfc.do)
+> 查询不好(单个查询限制500条，资助类别分类不准确)
+> 有2020年数据（不知道全不全）
+
+
+# [NSFC](http://output.nsfc.gov.cn/)
+> 资助项目查询已失效，只能查询结题项目

README.md

@@ -0,0 +1,4 @@
+find . -name __pycache__ -exec rm -rf {} \;
+
+rm -rf *.egg-info build dist
+

build.sh

@@ -0,0 +1,46 @@
+from pprint import pprint
+
+import click
+
+from nsfc.db.model import Project
+from nsfc.db.manager import Manager
+from nsfc.src.official import Official
+from nsfc.util.parse_data import parse
+
+
+@click.command()
+@click.option('-i', '--infile', help='the input filename', required=True)
+@click.option('-d', '--dbfile', help='the path of database file', default='project.db')
+@click.option('--echo', help='turn echo on for sqlalchemy', is_flag=True)
+@click.option('--drop', help='drop table before creating', is_flag=True)
+def build(**kwargs):
+    print(kwargs)
+    uri = 'sqlite:///{dbfile}'.format(**kwargs)
+    with Manager(uri=uri, echo=kwargs['echo'], drop=kwargs['drop']) as m:
+        for data in parse(kwargs['infile']):
+
+            query_result = m.query(Project, 'project_id', data['project_id']).first()
+            if query_result:
+                print('*** skip ***', query_result)
+                continue
+            
+            project = Project(**data)
+
+            conc_data = Official.get_conclusion_data(data['project_id'])
+            if conc_data:
+                project.finished = True
+                project.project_type_code = conc_data.get('projectType')
+                project.abstract = conc_data.get('projectAbstractC')
+                project.abstract_en = conc_data.get('projectAbstractE')
+                project.abstract_conc = conc_data.get('conclusionAbstract')
+                project.keyword = conc_data.get('projectKeywordC')
+                project.keyword_en = conc_data.get('projectKeywordE')
+                project.result_stat = conc_data.get('result_stat')
+
+            pprint(project.as_dict)
+            m.insert(Project, 'project_id', project)
+            break
+
+
+if __name__ == '__main__':
+    build()

nsfc/__init__.py

@@ -0,0 +1,61 @@
+import os
+import json
+
+import click
+
+from nsfc.src.letpub import LetPub
+from nsfc.src.official import Official
+
+from nsfc.db.model import Project
+from nsfc.db.manager import Manager
+from nsfc.util.parse_data import parse
+
+
+@click.command()
+@click.option('-y', '--year', help='the year of searching', required=True)
+@click.option('-c', '--code', help='the code of subject', required=True)
+@click.option('-O', '--outdir', help='the output directory')
+def main(**kwargs):
+    year = kwargs['year']
+    code = kwargs['code']
+    letpub = LetPub()
+    outdir = kwargs['outdir'] or os.path.join('done', code[0], str(year))
+    if not os.path.exists(outdir):
+        os.makedirs(outdir)
+    done = {each.rsplit('.', 1)[0]: 1 for each in os.listdir(outdir)}
+
+    for code in sorted(letpub.list_codes[code]):
+        if f'{code}.{year}' in done:
+            continue
+        outfile = f'{outdir}/{code}.{year}.jl'
+        try:
+            with open(outfile, 'w') as out:
+                for context in letpub.search(code, start_year=year, end_year=year):
+                    line = json.dumps(context, ensure_ascii=False) + '\n'
+                    out.write(line)
+            click.secho(f'save file: {outfile}')
+        except KeyboardInterrupt:
+            os.remove(outfile)
+            break
+
+
+if __name__ == '__main__':
+    main()
+
+# root_codes = Official.list_root_codes()
+# child_codes =  Official.list_child_codes('A')
+
+
+
+
+# done = {each.split('.')[0]: 1 for each in os.listdir('A')}
+
+# year = 2019
+# for child in child_codes:
+#     if child in done:
+#         # print(f'{child} is done')
+#         continue
+#     with open(f'{child}.{year}.jl', 'w') as out:
+#         for context in letpub.search(child, start_year=year, end_year=year):
+#             line = json.dumps(context, ensure_ascii=False) + '\n'
+#             out.write(line)