From 3b34ddcb247b3a62d674c85f150f4c263e2b45b4 Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 9 Jan 2019 00:07:54 -0600 Subject: [PATCH 01/15] Added Swagger API support and moved API to logic to API files. Also added few inital unit tests --- .gitignore | 2 + .travis.yml | 4 +- web/project/__init__.py | 10 +- web/project/__main__.py | 15 +- web/project/api.py | 62 +++ web/project/db.py | 4 +- web/project/models.py | 6 + web/project/paperdao.py | 10 +- web/project/routes.py | 97 ++-- .../static/javascript/modules/search.js | 4 +- web/project/swagger.yml | 150 +++++- web/project/templates/qrespexplorer.html | 10 +- web/project/tests/__init__.py | 0 .../tests/__pycache__/__init__.cpython-36.pyc | Bin 0 -> 149 bytes .../__pycache__/test_paperDAO.cpython-36.pyc | Bin 0 -> 2022 bytes .../__pycache__/test_routes.cpython-36.pyc | Bin 0 -> 914 bytes web/project/tests/data.json | 508 ++++++++++++++++++ web/project/tests/test_paperDAO.py | 72 +++ web/project/tests/test_routes.py | 24 + web/project/util.py | 34 ++ web/project/views.py | 4 + web/requirements.txt | 6 +- web/run.py | 6 +- web/setup.py | 6 +- 24 files changed, 930 insertions(+), 104 deletions(-) create mode 100644 web/project/api.py create mode 100644 web/project/tests/__init__.py create mode 100644 web/project/tests/__pycache__/__init__.cpython-36.pyc create mode 100644 web/project/tests/__pycache__/test_paperDAO.cpython-36.pyc create mode 100644 web/project/tests/__pycache__/test_routes.cpython-36.pyc create mode 100644 web/project/tests/data.json create mode 100644 web/project/tests/test_paperDAO.py create mode 100644 web/project/tests/test_routes.py diff --git a/.gitignore b/.gitignore index 3d637e3b..0ef78f98 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ web/flask_session/ flask_session/ web/dist/ web/project/__pycache__/ +web/build/ +web/qresp.egg-info/ diff --git a/.travis.yml b/.travis.yml index 65f88949..1ffd9fdd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,5 +9,5 @@ install: - cd web - python setup.py install # command to run tests -#script: -# - pytest +script: + - nose2 diff --git a/web/project/__init__.py b/web/project/__init__.py index f0cfc4d7..753b89c6 100644 --- a/web/project/__init__.py +++ b/web/project/__init__.py @@ -4,6 +4,7 @@ from flask_session import Session from flask_wtf import CSRFProtect from flask_sitemap import Sitemap +import connexion GOOGLE_CLIENT_ID = os.environ.get("GOOGLE_CLIENT_ID") GOOGLE_CLIENT_SECRET = os.environ.get("GOOGLE_CLIENT_SECRET") @@ -22,7 +23,13 @@ MONGODB_DB = os.environ.get("MONGODB_DB") -app = Flask(__name__) +#app = Flask(__name__) +# Create the application instance +connexionapp = connexion.App(__name__, specification_dir='./') + +# Read the swagger.yml file to configure the endpoints +connexionapp.add_api('swagger.yml') +app = connexionapp.app app.secret_key = '\\\xfcS\x1e\x8f\xfb]6\x1e.\xa8\xb3\xe1x\xc8\x8e\xc1\xeb5^x\x81\xcc\xd5' csrf = CSRFProtect(app) SESSION_TYPE = 'filesystem' @@ -47,5 +54,4 @@ Session(app) ext = Sitemap(app) - from project import routes diff --git a/web/project/__main__.py b/web/project/__main__.py index e0c7c7c3..859efd5a 100644 --- a/web/project/__main__.py +++ b/web/project/__main__.py @@ -2,6 +2,17 @@ # For desktop version running from command line from flask import Flask -app = Flask(__name__,template_folder='project/templates') +import connexion +#app = Flask(__name__) +# Create the application instance +connexionapp = connexion.App(__name__, specification_dir='./',template_folder='project/templates') + +# Read the swagger.yml file to configure the endpoints +connexionapp.add_api('swagger.yml') +app = connexionapp.app from .routes import main -main() \ No newline at end of file +import sys +port = 80 +if len(sys.argv)>1: + port = sys.argv[1] +main(port) \ No newline at end of file diff --git a/web/project/api.py b/web/project/api.py new file mode 100644 index 00000000..6bd87612 --- /dev/null +++ b/web/project/api.py @@ -0,0 +1,62 @@ +from .paperdao import * +def search(searchWord=None,paperTitle=None,doi=None,tags=None,collectionList=[],authorsList=[],publicationList=[]): + """ + This function responds to a request for /api/search + with the complete lists of papers + + :return: list of papers + """ + allpaperslist = [] + try: + dao = PaperDAO() + allpaperslist = dao.getAllFilteredSearchObjects(searchWord=searchWord,paperTitle=paperTitle,doi=doi, + tags=tags,collectionList=collectionList, + authorsList=authorsList,publicationList=publicationList) + except Exception as e: + print("Exception in ", e) + return allpaperslist + +def collections(): + """ + This function responds to a request for /api/collections + with the complete lists of c + + :return: list of collections + """ + allcollectionlist = [] + try: + dao = PaperDAO() + allcollectionlist = dao.getCollectionList() + except Exception as e: + print("Exception in ", e) + return list(allcollectionlist) + +def authors(): + """ + This function responds to a request for /api/authors + with the complete lists of authors + + :return: list of authors + """ + allauthorlist = [] + try: + dao = PaperDAO() + allauthorlist = dao.getAuthorList() + except Exception as e: + print("Exception in ", e) + return list(allauthorlist) + +def publications(): + """ + This function responds to a request for /api/publications + with the complete lists of publications + + :return: list of publications + """ + allpublist = [] + try: + dao = PaperDAO() + allpublist = dao.getPublicationList() + except Exception as e: + print("Exception in ", e) + return list(allpublist) \ No newline at end of file diff --git a/web/project/db.py b/web/project/db.py index 36bb1a19..60ffc938 100644 --- a/web/project/db.py +++ b/web/project/db.py @@ -1,7 +1,7 @@ from flask_mongoengine import MongoEngine from pymongo import errors - -from project import app +from project import connexionapp +app = connexionapp.app class MongoDBConnection(): """Class representing connection to Mongo database diff --git a/web/project/models.py b/web/project/models.py index 934cc4eb..5a1516b4 100644 --- a/web/project/models.py +++ b/web/project/models.py @@ -307,6 +307,12 @@ def year(self): def year(self, val): self.__year = val + def __hash__(self): + return hash(self.__title) + + def __eq__(self, other): + return self.__title == other.__title + class PaperDetails(object): def __init__(self): diff --git a/web/project/paperdao.py b/web/project/paperdao.py index dca03198..0c54aa88 100644 --- a/web/project/paperdao.py +++ b/web/project/paperdao.py @@ -40,8 +40,8 @@ def getAllPapers(self): allPapers = [paper.to_json() for paper in Paper.objects()] return allPapers - def getAllFilteredSearchObjects(self, searchWord, paperTitle, doi, tags, collectionList, authorsList, - publicationList): + def getAllFilteredSearchObjects(self, searchWord=None, paperTitle=None, doi=None, tags=None, collectionList=[], authorsList=[], + publicationList=[]): """ Produces papers after filtering with word :param searchWord: @@ -58,13 +58,13 @@ def getAllFilteredSearchObjects(self, searchWord, paperTitle, doi, tags, collect allFilteredSearchObjects = self.__filtersearchedPaper(filteredPaper) else: searchquery = Q() - if paperTitle.strip(): + if paperTitle and paperTitle.strip(): searchTitleQuery = Q(reference__title__icontains=paperTitle) searchquery = searchquery & searchTitleQuery - if doi.strip(): + if doi and doi.strip(): searchdoiquery = Q(reference__DOI__icontains=doi) searchquery = searchquery & searchdoiquery - if tags.strip(): + if tags and tags.strip(): searchtagquery = Q(tags__icontains=tags) searchquery = searchquery & searchtagquery if len(collectionList) > 0: diff --git a/web/project/routes.py b/web/project/routes.py index bb38b807..cc07d913 100644 --- a/web/project/routes.py +++ b/web/project/routes.py @@ -7,7 +7,6 @@ import schedule from flask import render_template, request, flash, redirect, url_for, jsonify, session,send_file from project import csrf -from project import app from project import ext import smtplib @@ -16,7 +15,7 @@ from .paperdao import * from .util import * from .views import * - +from project import app qresp_config = {} ftp_dict = {} @@ -50,15 +49,6 @@ def closeConnection(): -@app.errorhandler(InvalidUsage) -def handle_invalid_usage(error): - """ Returns error code - """ - response = jsonify(error.to_dict()) - response.status_code = error.status_code - return response - - @app.before_request def make_session_permanent(): """ makes session values last for a day @@ -72,31 +62,8 @@ def make_session_permanent(): def index(): """ Fetches the homepage """ - dbAdmin = MongoDBConnection.getDB() return render_template('index.html') -@ext.register_generator -def index(): - # Not needed if you set SITEMAP_INCLUDE_RULES_WITHOUT_PARAMS=True - yield 'index', {} - yield 'qrespexplorer', {} - yield 'paperdetails', {'paperid': '594186a91bd40fd4f2ce1aa0'}, {} - yield 'paperdetails', {'paperid': '5bb12835096ace7a9d8eb97d'}, {} - yield 'paperdetails', {'paperid': '5b3441242986382db00fab92'}, {} - yield 'paperdetails', {'paperid': '5b33a02629863800f82e297c'}, {} - yield 'paperdetails', {'paperid': '5b339eb029863800f82e297b'}, {} - yield 'paperdetails', {'paperid': '5b0ef8381f87c63760acb909'}, {} - yield 'paperdetails', {'paperid': '5a5e4e933a2c122ca8e9cd92'}, {} - yield 'paperdetails', {'paperid': '5ad8df0650cefd3974d05a83'}, {} - yield 'paperdetails', {'paperid': '5a5e437e3a2c122ca8e9cd75'}, {} - yield 'paperdetails', {'paperid': '5950fbfc1bd40f6cb67fb939'}, {} - yield 'paperdetails', {'paperid': '5992072a7590617df5a88a85'}, {} - yield 'paperdetails', {'paperid': '5983afce759061384c1aae48'}, {} - yield 'paperdetails', {'paperid': '594c507f1bd40f5ebf185fde'}, {} - yield 'paperdetails', {'paperid': '594c50671bd40f5e9b5c043b'}, {} - yield 'paperdetails', {'paperid': '594c504a1bd40f5e7819a0aa'}, {} - - @app.route('/downloads/') def downloadfile(file=None): if file: @@ -109,14 +76,6 @@ def downloadfile(file=None): return jsonify(msg),400 - -@app.route('/qrespexplorer') -def qrespexplorer(): - """ Fetches the explorer homepage - """ - serverslist = Servers() - return render_template('qrespexplorer.html', serverslist=serverslist.getServersList()) - @app.route('/qrespcurator') def qrespcurator(): """ Fetches the curator homepage @@ -759,6 +718,25 @@ def acknowledgement(): ########################################EXPLORER############################################################################# +@csrf.exempt +@app.route('/qrespexplorer',methods=['GET','POST']) +def qrespexplorer(): + """ Fetches the explorer homepage + """ + dao = PaperDAO() + dao.getDB() + form = QrespServerForm() + serverslist = Servers() + form.serverList = [qrespserver['qresp_server_url'] for qrespserver in + serverslist.getServersList()] + form.serverList.append('http://localhost') + if request.method == 'POST': + if request.form.get('serversList'): + session["selectedserver"] = request.form.get('serversList') + else: + session['selectedserver'] = form.serverList + return redirect(url_for('search')) + return render_template('qrespexplorer.html', form=form) @csrf.exempt @app.route('/verifyPasscode',methods=['POST']) @@ -815,19 +793,21 @@ def search(): """ This method helps in filtering paper content :return: template: search.html """ + allpaperslist = [] try: - dao = PaperDAO() - allpaperslist = dao.getAllFilteredSearchObjects("", "", "", "", "", "", "") - if not allpaperslist: - allpaperslist = [] - return render_template('search.html', - collectionlist=dao.getCollectionList(), authorslist=dao.getAuthorList(), - publicationlist=dao.getPublicationList(), allPapersSize=len(dao.getAllSearchObjects()), - allpaperslist = allpaperslist) + selectedserver = session.get("selectedserver") + fetchdata = FetchDataFromAPI(selectedserver) + allpaperslist = fetchdata.fetchOutput('/api/search') + collectionlist = fetchdata.fetchOutput('/api/collections') + authorslist = fetchdata.fetchOutput('/api/authors') + publicationlist = fetchdata.fetchOutput('/api/publications') + allPapersSize = len(allpaperslist) except Exception as e: print(e) flash('Error in search. ' + str(e)) - return render_template('search.html',allpaperslist=[]) + return render_template('search.html', allpaperslist=allpaperslist) + return render_template('search.html',allpaperslist=allpaperslist,collectionlist=collectionlist,authorslist=authorslist, + publicationlist=publicationlist,allPapersSize=allPapersSize) # Fetches search word options after selecting server @@ -846,8 +826,12 @@ def searchWord(): collectionList = json.loads(request.args.get('collectionList')) authorsList = json.loads(request.args.get('authorsList')) publicationList = json.loads(request.args.get('publicationList')) - allpaperslist = dao.getAllFilteredSearchObjects(searchWord, paperTitle, doi, tags, collectionList, authorsList, - publicationList) + selectedserver = session.get("selectedserver") + fetchdata = FetchDataFromAPI(selectedserver) + url = '/api/search' + "?searchWord=" + searchWord + # url = '/api/search'+"?searchWord="+searchWord+"&paperTitle="+paperTitle+"&doi="+doi+"&tags="+tags+"&collectionList="+",".join(collectionList) + \ + # "&authorsList="+",".join(authorsList)+"&publicationList="+",".join(publicationList) + allpaperslist = fetchdata.fetchOutput(url) return jsonify(allpaperslist=allpaperslist) except Exception as e: print(e) @@ -897,6 +881,9 @@ def chartworkflow(): @csrf.exempt @app.route('/getDescriptor', methods=['POST','GET']) def getDescriptor(): + """ + Inserts and fetches the metadata into Papers collections + """ try: data = request.get_json() paper = data.get("metadata") @@ -927,7 +914,7 @@ def mint(): """ return render_template('mint.html') -def main(): +def main(port): app.secret_key = '\xfd{H\xe5<\x95\xf9\xe3\x96.5\xd1\x01Odownload"; var notebookPath = row["_Search__notebookPath"]; var notebookFile = row["_Search__notebookFile"]; - if (notebookPath !== "") { + if (notebookPath !==undefined && notebookPath !== "") { if (notebookPath.indexOf('notebooks') < 0) { notebookPath = notebookPath.replace(/^(https?:\/\/)?(www\.)?/, ''); notebookPath = "http://nbviewer.jupyter.org/url/" + notebookPath; } } - if (notebookFile !== undefined || notebookFile !== null || notebookFile !== "") { + if (notebookFile !== undefined && notebookFile !== null && notebookFile !== "") { notebookPath = notebookPath + "/" + notebookFile; } var notebooks = "
  • jupyter
  • "; diff --git a/web/project/swagger.yml b/web/project/swagger.yml index 7c97dc4d..a5983efb 100644 --- a/web/project/swagger.yml +++ b/web/project/swagger.yml @@ -1,7 +1,7 @@ swagger: "2.0" info: - description: This is the swagger file that goes with our server code - version: "1.0.0" + description: This is the swagger file that provides the API to fetch data from Qresp server + version: "1.0" title: Swagger REST Article consumes: - "application/json" @@ -10,25 +10,127 @@ produces: basePath: "/api" -## Paths supported by the server application -#paths: -# /people: -# get: -# operationId: "people.read" -# tags: -# - "People" -# summary: "The people data structure supported by the server application" -# description: "Read the list of people" -# responses: -# 200: -# description: "Successful read people list operation" -# schema: -# type: "array" -# items: -# properties: -# fname: -# type: "string" -# lname: -# type: "string" -# timestamp: -# type: "string" \ No newline at end of file +# Paths supported by the server application +paths: + /search: + get: + operationId: "project.api.search" + tags: + - "Papers" + summary: "Fetches all papers" + description: "Read the list of papers" + parameters: + - in: query + name: searchWord + type: string + required: false + description: Filter paper results based on search word + - in: query + name: paperTitle + type: string + required: false + description: Filter paper results based on paper title + - in: query + name: doi + type: string + required: false + description: Filter paper results based on paper doi + - in: query + name: tags + type: string + required: false + description: Filter paper results based on paper tags + - in: query + name: collectionList + type: array + required: false + description: Filter paper results based on collection + - in: query + name: authorsList + type: array + required: false + description: Filter paper results based on authors + - in: query + name: publicationList + required: false + type: array + description: Filter paper results based on publication + responses: + 200: + description: "Successful read papers list operation" + schema: + type: "array" + items: + properties: + _Search__abstract: + type: "string" + _Search__authors: + type: "string" + _Search__collections: + type: "array" + _Search__doi: + type: "string" + _Search__downloadPath: + type: "string" + _Search__fileServerPath: + type: "string" + _Search__folderAbsolutePath: + type: "string" + _Search__id": + type: "string" + _Search__notebookFile: + type: "string" + _Search__notebookPath: + type: "string" + _Search__publication: + type: "string" + _Search__serverPath: + type: "string" + _Search__tags: + type: "array" + _Search__title: + type: "string" + _Search__year: + type: "integer" + /collections: + get: + operationId: "project.api.collections" + tags: + - "Collections" + summary: "Fetches all collections" + description: "Read the list of collections" + responses: + 200: + description: "Successful read collections list operation" + schema: + type: "array" + items: + type: "string" + /authors: + get: + operationId: "project.api.authors" + tags: + - "Authors" + summary: "Fetches all authors" + description: "Read the list of authors" + responses: + 200: + description: "Successful read authors list operation" + schema: + type: "array" + items: + type: "string" + /publications: + get: + operationId: "project.api.publications" + tags: + - "Publications" + summary: "Fetches all publications" + description: "Read the list of publications" + responses: + 200: + description: "Successful read publications list operation" + schema: + type: "array" + items: + type: "string" \ No newline at end of file diff --git a/web/project/templates/qrespexplorer.html b/web/project/templates/qrespexplorer.html index 67d205c2..06dcff0a 100644 --- a/web/project/templates/qrespexplorer.html +++ b/web/project/templates/qrespexplorer.html @@ -5,11 +5,11 @@

    Select Qresp node to search

    -
    - + {% for server in form.serverList %} + {% endfor %} fL8(~V&Og2x~N1{i@12OutH0TL+;!3>&=ek&P@K*9*(my@$qOlWaxQE^OS zN@j^kOh9FFMq*KJKv8~HYH~?&Okh!JaY0OZYEn!AL@1^twHPQKAD@|*SrQ+wS5SG2 T!zMRBr8Fni4rF#Q5HkP()SV^d literal 0 HcmV?d00001 diff --git a/web/project/tests/__pycache__/test_paperDAO.cpython-36.pyc b/web/project/tests/__pycache__/test_paperDAO.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eacd236120d589fb5ed816e08d348a7f1957184f GIT binary patch literal 2022 zcmbtV&2Jnv6u0MlHyfHnp}p`iQlT&@By9mzDhRdRNWH)|rKzY!D~%>=Z!(kFnZfoF zy4sw!M+$!j|B|nq_!l_wo_8}@!g8ude)c>+zmMO0w!c_kZ@>BVt9}#^@+Vnp4$L1y zmn9HJ7%fPQqg9X*jVZ;sU08?nH9NMAj1xO%bYqt}%zaH_&;0wb&pqxZ-V?$+=D#M) z=X7Sr0dKM3)Dk%-d3#1cD}7F))?Z+g^dhQ*LoQV}l^o`DXMk*M7IX^zDRhaRG6LIZ z?++4L+DXN+)ZXbCjsjN=`TI%tLH~v1LiSUZsW1BnuLdWn7#)Zz=L04C&jgos|BN5? z>p82hz_>ItsneQ^{YOuC>sQ(V&83f~(GNleU`HC?kcIq*(B)@9IEg7EvBfB}01%tm z%z+W$VIGVw^H~6+$663WTX!(zgN3;PMv2XZ-iF?VE~h{g$tk1CV%Cg;M$n1+nE%G7 zMA^BMyM}r|{n?xosMd_mK>nPn_I!0pS%941H_k1!mUpJOYX1nSeMWvDFNs_u^}2nU}{|$#J804cOZ1Rgtd)lr7T{*G^ptrGqh~(EJB=D&<)vnD*F_ z;j{-;Q9v%Us?=^Kr7R{bbZa-4RT*8=o8O}&`;WU-Szd@8bG83i+gYi!RY@?CsuLXu zo)%{2IuAN8h8{blYmv$<}hE?3xWt zz1Xk^yQ!BDCW_3+Ii1p+!e~uNZYdkHJ;S4nht8VZX;OxYWta`av?#($P-Vz}$(AzP ziyR$Xl&anYkh)C*z5_wsh756`4#gPKR{$rn#Z7QV+bpg0CTPD@j>}h3xQ_LZ{73Km zph9{;h~ohylbinnV)dT4Lgv#e$P6nH*5jihn`cxu%YcaZ1Owaxrw=YA5DgK*g4hAF zKtbGwdAtQEEa_SW!;OoOu4w!i_Ale`(MlYyY-izryokF|@%fdAKUf)YBi_RcBuVgcfK%5_l2OIR=Gek{{ArvP z^A&Lm)C$xs_K@5~(mWjV8J0qwwv OSbP>-7uLqt0sRMpLsNyA9sPR`Py zKcxSpzkyrPDSshT?<6~QfC|v*jz_-t_)af(cDld6A7w|3kUwPW2_b(AuYCtW5k(7< zwqKf3s9C{gK^jnUN>rf2TcScur{O9{BOR;gD>KJXcUFYL*-s>i?*NC466yn4*EtrO z2dG=B=a6WE@F`PNv0IV`EqEHLNX1Y`ssmu$cRyBhr^nhlo6|KAei3U3uib^<$dsSggM%V3Cc1Q>D73n`_(@C*z5A2l>+IZWV>78H?p1bnoqv&Lc4n%n z&YjIZ7;Wopp+9GJyObfAZK5!!${yC&KC;@K*L{?Nl!V?4{z*P;nvaB|_(nJyzn0qw zN~noosma^WfzorQrFma1$^ldxii0hRP5*%fWgvLN@(hwCnSw!Rvn^*^IA(HEidtUk1L#^*O6Y*zJbiR07~8|XQ4)JDL@8%l2;UWA zR;hV`d0z-r(RK_+wkH_St1a?I2D)hi2v}JRJmw(}nl;34_yqiOtg@cZ%ZWp|zOx>G n>i{3a+EBvX5X$U9@FAZ3AD!oKTB*l4b=yTw%2}W7?{WGU1$)#8 literal 0 HcmV?d00001 diff --git a/web/project/tests/data.json b/web/project/tests/data.json new file mode 100644 index 00000000..2ffe18b2 --- /dev/null +++ b/web/project/tests/data.json @@ -0,0 +1,508 @@ +{ + "PIs":[ + { + "firstName":"Giulia", + "lastName":"Galli" + } + ], + "charts":[ + { + "caption":" Experimental PE spectrum of a 1 M solution of NaCl and molecular orbitals of a single water molecule corresponding to specific bands in the spectrum. The background due to secondary electrons has been subtracted following ref 11. The sharp peak at 12.6 eV arises from ionization of the 1b1 orbital of gas-phase water.", + "files":[ + "charts/figure1/winter_exp.txt" + ], + "id":"c0", + "imageFile":"charts/figure1/fig1.png", + "kind":"figure", + "notebookFile":"charts/figure1/plot.ipynb", + "number":"1", + "properties":[ + "electron binding energy", + "photo-electron spectrum" + ] + }, + { + "caption":"PE spectra of a 1 M aqueous NaCl solution computed with density functional approximations compared to the experimental spectrum of Figure 1. The gray area under the spectral features of sodium (2p) and chloride (3p) is the DOS projected on the maximally localized Wannier functions centered on the ionic sites. Theoretical spectra were aligned to the vacuum level (see text and Table SI).", + "files":[ + "charts/figure2/winter_exp.txt", + "charts/figure2/pdos_pbe.txt", + "charts/figure2/pdos_pbe0.txt", + "charts/figure2/pdos_rsh.txt", + "charts/figure2/pdos_sc.txt" + ], + "id":"c1", + "imageFile":"charts/figure2/fig2.png", + "kind":"figure", + "notebookFile":"charts/figure2/plot.ipynb", + "number":"2", + "properties":[ + "density of states", + "photo-electron spectrum" + ] + }, + { + "caption":"PE spectra of a 1 M aqueous NaCl solution computed using the G0W0 approximation starting from different sets of eigenfunctions and eigenvalues. Each theoretical spectrum is from a single snapshot representative of the entire trajectory. The spectra were aligned to the vacuum level (see text and SI). The gray area is projected DOS on Na and Cl atoms defined in Figure 2 caption.", + "files":[ + "charts/figure3/winter_exp.txt", + "charts/figure3/pdos_pbe_GW.txt", + "charts/figure3/pdos_pbe0_GW.txt", + "charts/figure3/pdos_rsh_GW.txt", + "charts/figure3/pdos_sc_GW.txt" + ], + "id":"c2", + "imageFile":"charts/figure3/fig3.png", + "kind":"figure", + "notebookFile":"charts/figure3/plot.ipynb", + "number":"3", + "properties":[ + "density of states", + "photo-electron spectrum" + ] + }, + { + "caption":"Experimental (black) and theoretical (blue) PE spectra of a 1 M aqueous solution of NaCl. Theoretical spectrum was computed using G0W0/sc-hybrid line widths and experimental photoionization cross sections as explained in the SI. Both spectra were normalized to the 1b1 peak of water. The gray area is projected DOS on Na and Cl atoms (see Figure 2 caption).", + "files":[ + "charts/figure4/winter_exp.txt", + "charts/figure4/pdos_GW_Banna_115eV+Yeh_ions_200eV.txt", + "charts/figure4/pdos_pbe0_GW.txt", + "charts/figure4/pdos_rsh_GW.txt", + "charts/figure4/pdos_sc_GW.txt" + ], + "id":"c3", + "imageFile":"charts/figure4/fig4.png", + "kind":"figure", + "notebookFile":"charts/figure4/plot.ipynb", + "number":"4", + "properties":[ + "density of states", + "photo-electron spectrum" + ] + }, + { + "caption":"Electron BE in PE Spectra of a 1 M NaCl Solution Computed Using DFT and Many-Body Perturbation Theory at the G0W0 Level", + "files":[ + "charts/table1/dos.ods" + ], + "id":"c4", + "imageFile":"charts/table1/table1.png", + "kind":"table", + "notebookFile":"charts/table1/plot.ipynb", + "number":"1", + "properties":[ + "electron binding energy", + "photo-electron spectrum" + ] + } + ], + "collections":[ + "MICCoM" + ], + "datasets":[ + { + "files":[ + "charts/figure1" + ], + "id":"d0", + "readme":"Processed experimental spectrum from Winters and Siedel and orbital image files -- processing of experimental data unknown" + }, + { + "files":[ + "data/input" + ], + "id":"d1", + "readme":"Example input files for dft pw.x calculations, MLWF calculations and WEST G0W0 calculations." + }, + { + "files":[ + "data/s10" + ], + "id":"d2", + "readme":"electronic structure outputs at various levels of theory (pbe, pbe0, rsh, sc-hybrid, G0W0@pbe, G0W0@pbe0, G0W0@rsh, G0W0@sc-hybrid) along with MLWF outputs at the same levels of theory. Each subfolder geometry in the s10 subfolder contains a geometry obtained as a snapshot sxxxx from a PBE0 MD trajectory of 1M salt water." + }, + { + "files":[ + "data/s15" + ], + "id":"d3", + "readme":"electronic structure outputs at various levels of theory (pbe, pbe0, rsh, sc-hybrid, G0W0@pbe, G0W0@pbe0, G0W0@rsh, G0W0@sc-hybrid) along with MLWF outputs at the same levels of theory. Each subfolder geometry in the s15 subfolder contains a geometry obtained as a snapshot sxxxx from a PBE0 MD trajectory of 1M salt water." + } + ], + "heads":[ + { + "URLs":[ + "" + ], + "id":"h0", + "readme":"Experimental setup info" + }, + { + "URLs":[ + "http://www.quantum-simulation.org/reference/nacl/index.htm" + ], + "id":"h1", + "readme":"NACL2016 dataset: NVT s10 simulation data" + }, + { + "URLs":[ + "http://www.quantum-simulation.org/reference/nacl/index.htm" + ], + "id":"h2", + "readme":"NACL2016 dataset: NVT s15 simulation data" + } + ], + "info":{ + "downloadPath":"https://www.globus.org/app/transfer?origin_id=72277ed4-1ad3-11e7-bbe1-22000b9a448b&origin_path=jacs.6b00225", + "fileServerPath":"http://notebook.rcc.uchicago.edu/files/jacs.6b00225", + "folderAbsolutePath":"/cds/gagalli/DATA_COLLECTIONS/public/jacs.6b00225", + "insertedBy":{ + "firstName":"Jonathan", + "lastName":"Skone", + "middleName":"H." + }, + "isPublic":true, + "notebookFile":"Main.ipynb", + "notebookPath":"http://notebook.rcc.uchicago.edu/notebooks/jacs.6b00225", + "serverPath":"midway.rcc.uchicago.edu", + "timeStamp":"2017-06-22 18:19:02" + }, + "reference":{ + "DOI":"10.1021/jacs.6b00225", + "authors":[ + { + "firstName":"Alex", + "lastName":"Gaiduk" + }, + { + "firstName":"Marco", + "lastName":"Govoni" + }, + { + "firstName":"Robert", + "lastName":"Siedel" + }, + { + "firstName":"Jonathan", + "lastName":"Skone", + "middleName":"H." + }, + { + "firstName":"Bernd", + "lastName":"Winter" + }, + { + "firstName":"Giulia", + "lastName":"Galli" + } + ], + "journal":{ + "abbrevName":"JACS Comm.", + "fullName":"Journal of the American Chemical Society" + }, + "kind":"article", + "page":"6912", + "publishedAbstract":"We present a combined computational and experimental study of the photoelectron spectrum of a simple aqueous solution of NaCl. Measurements were conducted on microjets, and first-principles calculations were performed using hybrid functionals and many-body perturbation theory at the G0W0 level, starting with wave functions computed in ab initio molecular dynamics simulations. We show excellent agreement between theory and experiments for the positions of both the solute and solvent excitation energies on an absolute energy scale and for peak intensities. The best comparison was obtained using wave functions obtained with dielectric-dependent self-consistent and range-separated hybrid functionals. Our computational protocol opens the way to accurate, predictive calculations of the electronic properties of electrolytes, of interest to a variety of energy problems.", + "publishedDate":"2016-04-22 00:00:00", + "receivedDate":"2016-01-08 00:00:00", + "title":"Photoelectron spectra of aqueous solutions from first principle", + "volume":"138", + "year":2016 + }, + "scripts":[ + { + "files":[ + "charts/figure1/pes.plt" + ], + "id":"s0", + "readme":"gnuplot script to generate experimental PES plot of water overlayed with the orbitals for each DOS peak" + }, + { + "files":[ + "scripts/coords.sh" + ], + "id":"s1", + "readme":"extract atomic coordinates from Qbox output files" + }, + { + "files":[ + "scripts/mksam.sh" + ], + "id":"s2", + "readme":"make sample" + }, + { + "files":[ + "scripts/pdos.sh" + ], + "id":"s3", + "readme":"evaluate projected-density of states from wannier localized orbitals" + }, + { + "files":[ + "scripts/average.sh" + ], + "id":"s4", + "readme":"compute average of some quantity" + }, + { + "files":[ + "scripts/pdos-GW.sh" + ], + "id":"s5", + "readme":"projected density of states for GW calculations" + }, + { + "files":[ + "scripts/pdos+intensities.sh" + ], + "id":"s6", + "readme":"evaluate the projected density of states intensities" + }, + { + "files":[ + "scripts/detmax.sh" + ], + "id":"s7", + "readme":"determine maximum peak location" + }, + { + "files":[ + "charts/figure2/dft.plt" + ], + "id":"s8", + "readme":"gnuplot script to plot the experimental PES of salt water overlayed with theoretical DOS evaluated at several DFT levels of theory" + }, + { + "files":[ + "charts/figure3/gw.plt" + ], + "id":"s9", + "readme":"gnuplot script to plot the experimental PES of salt water overlayed with theoretical DOS evaluated at GW@dft level of theory where dft is several dft levels of theory" + }, + { + "files":[ + "charts/figure4/sc_intensity_Banna_115eV.plt" + ], + "id":"s10", + "readme":"gnuplot script to plot the experimental PES of salt water overlayed with theoretical DOS evaluated at GW@sc-hybrid level of theory" + } + ], + "tags":[ + "DFT", + "MBPT", + "GW", + "PES", + "photo electron spectra", + "band edges", + "density of states", + "water", + "FPMD", + "Qbox", + "WEST" + ], + "tools":[ + { + "URLs":[ + "https://www.helmholtz-berlin.de/pubbin/igama_output?modus=einzel&sprache=en&gid=1648&typoid" + ], + "facilityName":"Helmholtz-Zentrum Berlin", + "id":"t0", + "kind":"experiment", + "measurement":"soft X-ray Photoemission" + }, + { + "URLs":[ + "http://www.quantum-espresso.org/" + ], + "id":"t1", + "kind":"software", + "packageName":"Quantum-Espresso", + "programName":"pw.x", + "version":"5.2" + }, + { + "URLs":[ + "http://www.west-code.org" + ], + "id":"t2", + "kind":"software", + "packageName":"West", + "programName":"wstat.x", + "version":"2.0.0" + }, + { + "URLs":[ + "http://www.west-code.org" + ], + "id":"t3", + "kind":"software", + "packageName":"West", + "programName":"wfreq.x", + "version":"2.0.0" + } + ], + "workflow":{ + "edges":[ + [ + "s4", + "s6" + ], + [ + "s4", + "s9" + ], + [ + "s4", + "s8" + ], + [ + "s4", + "s7" + ], + [ + "s9", + "c2" + ], + [ + "d3", + "s5" + ], + [ + "d3", + "s3" + ], + [ + "s5", + "s4" + ], + [ + "s2", + "d1" + ], + [ + "t3", + "d2" + ], + [ + "t3", + "d3" + ], + [ + "s0", + "c0" + ], + [ + "h1", + "s1" + ], + [ + "s6", + "s10" + ], + [ + "s8", + "c1" + ], + [ + "s3", + "s4" + ], + [ + "d2", + "s5" + ], + [ + "d2", + "s3" + ], + [ + "h0", + "t0" + ], + [ + "s1", + "s2" + ], + [ + "d0", + "s10" + ], + [ + "d0", + "s9" + ], + [ + "d0", + "s0" + ], + [ + "d0", + "s8" + ], + [ + "t1", + "d2" + ], + [ + "t1", + "t2" + ], + [ + "t1", + "d3" + ], + [ + "s7", + "c4" + ], + [ + "d1", + "t1" + ], + [ + "s10", + "c3" + ], + [ + "h2", + "s1" + ], + [ + "t0", + "d0" + ], + [ + "t2", + "t3" + ] + ], + "nodes":[ + "s4", + "s9", + "d3", + "s5", + "s2", + "t3", + "s0", + "h1", + "s6", + "s8", + "s3", + "d2", + "h0", + "c4", + "s1", + "d0", + "c1", + "t1", + "s7", + "c0", + "d1", + "c2", + "s10", + "h2", + "t0", + "c3", + "t2" + ] + } +} \ No newline at end of file diff --git a/web/project/tests/test_paperDAO.py b/web/project/tests/test_paperDAO.py new file mode 100644 index 00000000..f7b0ac5f --- /dev/null +++ b/web/project/tests/test_paperDAO.py @@ -0,0 +1,72 @@ +import unittest +from unittest import TestCase +from project.paperdao import * +import os +import mongomock +import json +def warn(*args, **kwargs): + pass +import warnings +warnings.warn = warn + +class TestPaperDAO(TestCase): + def setUp(self): + """ + Sets up database to test + """ + MongoDBConnection.getDB(hostname='mongomock://localhost', port=int('27017'), + username=None, password=None, + dbname='mongoenginetest', collection='paper', + isssl='No') + __location__ = os.path.realpath( + os.path.join(os.getcwd(), os.path.dirname(__file__))) + with open(os.path.join(__location__, 'data.json')) as f: + paperdata = json.load(f) + #paperdata = json.loads(paperdata) + paper = Paper(**paperdata) + paper.save() + + def test_getCollectionList(self): + """ + Tests if all collections exist + """ + dao = PaperDAO() + allcollectionlist = dao.getCollectionList() + self.assertTrue(list(allcollectionlist)) + + def test_getPublicationList(self): + """ + Tests for publications + """ + dao = PaperDAO() + allpublicationlist = dao.getPublicationList() + self.assertTrue(list(allpublicationlist)) + + def test_getAllPapers(self): + """ + Tests for all papers + """ + dao = PaperDAO() + allpapers = dao.getAllPapers() + self.assertTrue(list(allpapers)) + + # def test_getAllFilteredSearchObjects(self): + # self.fail() + # + # def test_getAllSearchObjects(self): + # self.fail() + # + # def test_insertIntoPapers(self): + # self.fail() + # + # def test_getPaperDetails(self): + # self.fail() + # + # def test_getWorkflowDetails(self): + # self.fail() + # + # def test_getWorkflowForChartDetails(self): + # self.fail() + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/web/project/tests/test_routes.py b/web/project/tests/test_routes.py new file mode 100644 index 00000000..2877aa63 --- /dev/null +++ b/web/project/tests/test_routes.py @@ -0,0 +1,24 @@ +import os +import unittest + +from project import app + +class RouteTests(unittest.TestCase): + ############################ + #### setup and teardown #### + ############################ + + # executed prior to each test + def setUp(self): + self.app = app.test_client() + + # executed after each test + def tearDown(self): + pass + + def test_main_page(self): + response = self.app.get('/', follow_redirects=True) + self.assertEqual(response.status_code, 200) + +if __name__ == "__main__": + unittest.main() diff --git a/web/project/util.py b/web/project/util.py index c202fc28..dce5b495 100644 --- a/web/project/util.py +++ b/web/project/util.py @@ -6,6 +6,7 @@ import requests from jsonschema import Draft4Validator from requests_oauthlib import OAuth2Session +import itertools from .views import ReferenceForm, ChartForm, DatasetForm, ToolForm, ScriptForm, HeadForm @@ -400,6 +401,39 @@ def sendDescriptorToServer(self,data): response = requests.post(url, data=json.dumps(payload), headers=headers, verify=False) return response +class FetchDataFromAPI(): + """ Fetches data for search,paper details and chart details for explorer + """ + def __init__(self,servernames): + if servernames and len(servernames)>0: + self.__servernames = servernames + else: + serverslist = Servers() + serverList = [qrespserver['qresp_server_url'] for qrespserver in + serverslist.getServersList()] + serverList.append('http://localhost') + serverList.append('http://localhost') + self.__servernames = serverList + + def fetchOutput(self,apiname): + """ Fetches output to server + :return: object: sends descriptor to server + """ + outDict = {} + for snames in self.__servernames: + url = snames + apiname + print("url>",url) + headers = {'Application': 'qresp', 'Accept': 'application/json', 'Content-Type': 'application/json'} + response = requests.get(url,headers=headers, verify=False) + if response.status_code == 200: + if "search" in apiname: + outDict.update({eachsearchObj['_Search__title']:eachsearchObj for eachsearchObj in response.json()}) + else: + outDict.update({eachsearchObj:eachsearchObj for eachsearchObj in response.json()}) + output = list(outDict.values()) + return output + + class GoogleAuth(): """ Authorization for google. """ diff --git a/web/project/views.py b/web/project/views.py index ef7e2580..fdc173b1 100644 --- a/web/project/views.py +++ b/web/project/views.py @@ -48,6 +48,10 @@ class ConfigForm(Form): downloadPath = StringField('Download Path',description='The Globus service allows to download the paper content using gridFTP.') fileServerPath = StringField('File Server Path', description='If a HTTP service is running on the server, a URL may be associated to the paper content. This URL is only required by Qresp | Exploration to view images & download files using a web browser.') +class QrespServerForm(Form): + serverList = FieldList(StringField(description='Select the Address of the Database. Qresp automatically inserts the metadata file in the database')) + + class NameForm(Form): firstName = StringField(validators=[validators.DataRequired()],description='e.g. John',render_kw={"placeholder": "Enter first name"}) middleName = StringField(description='e.g. L.',render_kw={"placeholder": "Enter middle name"}) diff --git a/web/requirements.txt b/web/requirements.txt index e7e9c80d..31d7a5a8 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -22,4 +22,8 @@ Werkzeug==0.14.1 WTForms==2.2.1 schedule flask-sitemap -requests_oauthlib \ No newline at end of file +requests_oauthlib +mongomock +connexion +coverage +nose2 \ No newline at end of file diff --git a/web/run.py b/web/run.py index 40913e57..a5b9fff3 100644 --- a/web/run.py +++ b/web/run.py @@ -1,11 +1,11 @@ # run.py from flask_cors import CORS -from project import app - +from project import connexionapp +app = connexionapp.app CORS(app) #app.run() if __name__ == "__main__": - app.run(port=80,debug=True) + connexionapp.run(port=80,debug=True) diff --git a/web/setup.py b/web/setup.py index 23db8d21..e03d0305 100644 --- a/web/setup.py +++ b/web/setup.py @@ -35,7 +35,11 @@ 'schedule', 'wtforms', 'flask-sitemap', - 'requests_oauthlib' + 'requests_oauthlib', + 'mongomock', + 'connexion', + 'coverage', + 'nose2' ], include_package_data=True ) From aba57d8a284d1c85c932b989795fcc3c1425a6af Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 9 Jan 2019 12:11:10 -0600 Subject: [PATCH 02/15] Updated swagger API tests --- web/MANIFEST.in | 3 ++- web/project/__main__.py | 3 +-- web/project/routes.py | 8 +++----- web/project/swagger.yml | 6 +++--- web/project/tests/test_paperDAO.py | 1 - web/project/tests/test_routes.py | 4 ++++ web/project/util.py | 18 +++++++++++++----- 7 files changed, 26 insertions(+), 17 deletions(-) diff --git a/web/MANIFEST.in b/web/MANIFEST.in index eeb65b00..628a7319 100644 --- a/web/MANIFEST.in +++ b/web/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include project/templates * -recursive-include project/static * \ No newline at end of file +recursive-include project/static * +include project/swagger.yml diff --git a/web/project/__main__.py b/web/project/__main__.py index 859efd5a..9ec8bb4f 100644 --- a/web/project/__main__.py +++ b/web/project/__main__.py @@ -3,9 +3,8 @@ from flask import Flask import connexion -#app = Flask(__name__) # Create the application instance -connexionapp = connexion.App(__name__, specification_dir='./',template_folder='project/templates') +connexionapp = connexion.App(__name__, specification_dir='./') # Read the swagger.yml file to configure the endpoints connexionapp.add_api('swagger.yml') diff --git a/web/project/routes.py b/web/project/routes.py index cc07d913..9d560dee 100644 --- a/web/project/routes.py +++ b/web/project/routes.py @@ -19,9 +19,6 @@ qresp_config = {} ftp_dict = {} - - - class InvalidUsage(Exception): """ Invalid page """ @@ -62,6 +59,8 @@ def make_session_permanent(): def index(): """ Fetches the homepage """ + #TO BE DELETED + SetLocalHost(str(request.host_url).strip("/")) return render_template('index.html') @app.route('/downloads/') @@ -729,7 +728,6 @@ def qrespexplorer(): serverslist = Servers() form.serverList = [qrespserver['qresp_server_url'] for qrespserver in serverslist.getServersList()] - form.serverList.append('http://localhost') if request.method == 'POST': if request.form.get('serversList'): session["selectedserver"] = request.form.get('serversList') @@ -764,11 +762,11 @@ def admin(): dbAdmin = MongoDBConnection.getDB(hostname=form.hostname.data, port=int(form.port.data), username=form.username.data, password=form.password.data, dbname=form.dbname.data, collection=form.collection.data, isssl=form.isSSL.data) - paperCollectionlist = Paper.objects.get_unique_values('collections') except Exception as e: flash('Could not connect to server, \n ' + str(e)) raise InvalidUsage('Could not connect to server, \n ' + str(e), status_code=410) flash('Connected') + SetLocalHost(str(request.host_url).strip("/")) return redirect(url_for('qrespexplorer')) return render_template('admin.html', form=form,verifyform =verifyform ) diff --git a/web/project/swagger.yml b/web/project/swagger.yml index a5983efb..4db13a1b 100644 --- a/web/project/swagger.yml +++ b/web/project/swagger.yml @@ -98,7 +98,7 @@ paths: tags: - "Collections" summary: "Fetches all collections" - description: "Read the list of collections" + description: "Fetches collections from all Papers" responses: 200: description: "Successful read collections list operation" @@ -112,7 +112,7 @@ paths: tags: - "Authors" summary: "Fetches all authors" - description: "Read the list of authors" + description: "Fetches authors from papers" responses: 200: description: "Successful read authors list operation" @@ -126,7 +126,7 @@ paths: tags: - "Publications" summary: "Fetches all publications" - description: "Read the list of publications" + description: "Fetches publications from all papers" responses: 200: description: "Successful read publications list operation" diff --git a/web/project/tests/test_paperDAO.py b/web/project/tests/test_paperDAO.py index f7b0ac5f..f204803d 100644 --- a/web/project/tests/test_paperDAO.py +++ b/web/project/tests/test_paperDAO.py @@ -22,7 +22,6 @@ def setUp(self): os.path.join(os.getcwd(), os.path.dirname(__file__))) with open(os.path.join(__location__, 'data.json')) as f: paperdata = json.load(f) - #paperdata = json.loads(paperdata) paper = Paper(**paperdata) paper.save() diff --git a/web/project/tests/test_routes.py b/web/project/tests/test_routes.py index 2877aa63..c0e08f65 100644 --- a/web/project/tests/test_routes.py +++ b/web/project/tests/test_routes.py @@ -20,5 +20,9 @@ def test_main_page(self): response = self.app.get('/', follow_redirects=True) self.assertEqual(response.status_code, 200) + def test_main_page(self): + response = self.app.get('/', follow_redirects=True) + self.assertEqual(response.status_code, 200) + if __name__ == "__main__": unittest.main() diff --git a/web/project/util.py b/web/project/util.py index dce5b495..5192413a 100644 --- a/web/project/util.py +++ b/web/project/util.py @@ -12,6 +12,7 @@ from .views import ReferenceForm, ChartForm, DatasetForm, ToolForm, ScriptForm, HeadForm +LOCALHOST = None @@ -406,14 +407,16 @@ class FetchDataFromAPI(): """ def __init__(self,servernames): if servernames and len(servernames)>0: - self.__servernames = servernames + serverList = servernames else: serverslist = Servers() serverList = [qrespserver['qresp_server_url'] for qrespserver in serverslist.getServersList()] - serverList.append('http://localhost') - serverList.append('http://localhost') - self.__servernames = serverList + global LOCALHOST + print(LOCALHOST) + if LOCALHOST: + serverList.append(LOCALHOST) + self.__servernames = serverList def fetchOutput(self,apiname): """ Fetches output to server @@ -453,4 +456,9 @@ def getGoogleAuth(self,state=None,token=None): state=state, redirect_uri=self.redirecturi) oauth = OAuth2Session(self.clientid,redirect_uri=self.redirecturi,scope=self.scope) - return oauth \ No newline at end of file + return oauth + +class SetLocalHost: + def __init__(self,localhost): + global LOCALHOST + LOCALHOST = localhost \ No newline at end of file From 79fea1028084530f8ca1dc99362dd7963239044c Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 9 Jan 2019 13:40:38 -0600 Subject: [PATCH 03/15] Updated Travis --- web/project/__init__.py | 11 +++++------ web/project/tests/test_paperDAO.py | 26 +++++++++++++++++--------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/web/project/__init__.py b/web/project/__init__.py index 753b89c6..b843c590 100644 --- a/web/project/__init__.py +++ b/web/project/__init__.py @@ -45,12 +45,11 @@ os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' app.config['MAIL_ADDR'] = MAIL_ADDR app.config['MAIL_PWD'] = MAIL_PWD -app.config['MONGODB_HOST'] = MONGODB_HOST -app.config['MONGODB_PORT'] = MONGODB_PORT -app.config['MONGODB_USERNAME'] = MONGODB_USERNAME -app.config['MONGODB_PASSWORD'] = MONGODB_PASSWORD -app.config['MONGODB_DB'] = MONGODB_DB - +app.config['MONGODB_HOST'] = 'paperstack.uchicago.edu' +app.config['MONGODB_PORT'] = 27017 +app.config['MONGODB_USERNAME'] = 'qresp_user_explorer' +app.config['MONGODB_PASSWORD'] = 'qresp_pwd' +app.config['MONGODB_DB'] = 'explorer' Session(app) ext = Sitemap(app) diff --git a/web/project/tests/test_paperDAO.py b/web/project/tests/test_paperDAO.py index f204803d..6e5e096b 100644 --- a/web/project/tests/test_paperDAO.py +++ b/web/project/tests/test_paperDAO.py @@ -49,15 +49,23 @@ def test_getAllPapers(self): allpapers = dao.getAllPapers() self.assertTrue(list(allpapers)) - # def test_getAllFilteredSearchObjects(self): - # self.fail() - # - # def test_getAllSearchObjects(self): - # self.fail() - # - # def test_insertIntoPapers(self): - # self.fail() - # + def test_getAllFilteredSearchObjects(self): + """ + Tests for all search Objects + """ + dao = PaperDAO() + allSearchObjects = dao.getAllFilteredSearchObjects() + self.assertTrue(list(allSearchObjects)) + + def test_getFilteredPaperObjects(self): + """ + Tests for all search Objects with name + """ + dao = PaperDAO() + allSearchObjects = dao.getAllFilteredSearchObjects(authorsList=['Marco Govoni']) + print(allSearchObjects) + self.assertTrue(list(allSearchObjects)) + # def test_getPaperDetails(self): # self.fail() # From 044a5022ae914bf417861eaa7eb352fcb598b173 Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 9 Jan 2019 13:57:21 -0600 Subject: [PATCH 04/15] Updated setup for travis --- web/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/web/setup.py b/web/setup.py index e03d0305..4c041458 100644 --- a/web/setup.py +++ b/web/setup.py @@ -26,7 +26,6 @@ 'mongoengine', 'cryptography', 'jinja2', - 'requests', 'pyOpenSSL', 'werkzeug', 'itsdangerous', From 371d248122e34c4eff11e028245f7cbe8d329c49 Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 9 Jan 2019 14:06:09 -0600 Subject: [PATCH 05/15] Updated setup for travis --- .travis.yml | 1 + web/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1ffd9fdd..3b267d95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: # command to install dependencies install: - cd web + - pip install -r requirements.txt - python setup.py install # command to run tests script: diff --git a/web/requirements.txt b/web/requirements.txt index 31d7a5a8..fe657a5e 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -1,3 +1,4 @@ +setuptools Flask==1.0.2 Flask-API==1.0 Flask-Cors==3.0.6 From 7e22acd510d566009a19687cef2923aa0bcaa307 Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 9 Jan 2019 14:12:55 -0600 Subject: [PATCH 06/15] Updated travis and setup file --- .travis.yml | 1 - web/setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3b267d95..0f604c96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "3.3" - "3.4" - "3.5" - "3.6" diff --git a/web/setup.py b/web/setup.py index 4c041458..041e8dad 100644 --- a/web/setup.py +++ b/web/setup.py @@ -11,7 +11,7 @@ author='Aditya Tanikanti, Marco Govoni', author_email='datadev@lists.uchicago.edu', description='Qresp "Curation and Exploration of Reproducible Scientific Papers" is a Python application that facilitates the organization, annotation and exploration of data presented in scientific papers. ', - python_requires='>=3.0', + python_requires='>=3.4', packages=find_packages(), install_requires=[ 'flask_api', From 697f9ff2b782ac851e2743836852ce126e357288 Mon Sep 17 00:00:00 2001 From: atanikan Date: Fri, 11 Jan 2019 09:33:06 -0600 Subject: [PATCH 07/15] Fixes for API and added more tests --- web/project/api.py | 63 ++++++++++++++++- web/project/paperdao.py | 58 ++++++++++++---- web/project/routes.py | 63 +++++++++++++---- .../static/javascript/modules/curate.js | 24 +++++-- .../static/javascript/modules/paperdetails.js | 12 ++-- web/project/swagger.yml | 69 ++++++++++++++++--- web/project/templates/curate.html | 2 +- web/project/templates/mint.html | 40 ++++------- web/project/templates/project.html | 2 +- web/project/templates/publish.html | 2 +- web/project/templates/qrespcurator.html | 2 +- web/project/tests/test_paperDAO.py | 52 +++++++++++++- web/project/util.py | 16 +++-- web/project/views.py | 2 +- web/requirements.txt | 2 +- web/setup.py | 2 +- 16 files changed, 322 insertions(+), 89 deletions(-) diff --git a/web/project/api.py b/web/project/api.py index 6bd87612..225e7be4 100644 --- a/web/project/api.py +++ b/web/project/api.py @@ -1,5 +1,5 @@ from .paperdao import * -def search(searchWord=None,paperTitle=None,doi=None,tags=None,collectionList=[],authorsList=[],publicationList=[]): +def search(searchWord=None,paperTitle=None,doi=None,tags=None,collectionList=None,authorsList=None,publicationList=None): """ This function responds to a request for /api/search with the complete lists of papers @@ -9,11 +9,17 @@ def search(searchWord=None,paperTitle=None,doi=None,tags=None,collectionList=[], allpaperslist = [] try: dao = PaperDAO() + if collectionList: + collectionList = collectionList.split(",") + if authorsList: + authorsList = authorsList.split(",") + if publicationList: + publicationList = publicationList.split(",") allpaperslist = dao.getAllFilteredSearchObjects(searchWord=searchWord,paperTitle=paperTitle,doi=doi, tags=tags,collectionList=collectionList, authorsList=authorsList,publicationList=publicationList) except Exception as e: - print("Exception in ", e) + print("Exception in search", e) return allpaperslist def collections(): @@ -59,4 +65,55 @@ def publications(): allpublist = dao.getPublicationList() except Exception as e: print("Exception in ", e) - return list(allpublist) \ No newline at end of file + return list(allpublist) + +def paper(id): + """ + This function responds to a request for /api/paper/{id} + with the details of paper given id + + :return: paper details object + """ + paperdetail = None + try: + dao = PaperDAO() + paperdetail = dao.getPaperDetails(id) + except Exception as e: + msg = "Exception in paper api " + str(e) + print("Exception in paper api ", e) + return e, 500 + return paperdetail + +def workflow(id): + """ + This function responds to a request for /api/workflow/{id} + with the workflow given id + + :return: workflow object + """ + workflowdetail = None + try: + dao = PaperDAO() + workflowdetail = dao.getWorkflowDetails(id) + except Exception as e: + msg = "Exception in workflow api " + str(e) + print("Exception in workflow api ", e) + return msg,500 + return workflowdetail + +def chart(id,cid): + """ + This function responds to a request for /api/paper/{id}/chart/{cid} + with the chart given id + + :return: chart object + """ + chartworkflowdetail = None + try: + dao = PaperDAO() + chartworkflowdetail = dao.getWorkflowForChartDetails(id, cid) + except Exception as e: + msg = "Exception in chart api " + str(e) + print("Exception in chart api ", e) + return msg,500 + return chartworkflowdetail \ No newline at end of file diff --git a/web/project/paperdao.py b/web/project/paperdao.py index 0c54aa88..e7fa7710 100644 --- a/web/project/paperdao.py +++ b/web/project/paperdao.py @@ -53,29 +53,55 @@ def getAllFilteredSearchObjects(self, searchWord=None, paperTitle=None, doi=None searchRegWord = re.compile(regSearch, re.IGNORECASE) filteredPaper = Paper.objects.filter( Q(reference__title=searchRegWord) | Q(reference__publishedAbstract=searchRegWord) | - Q(tags=searchRegWord) | Q(collections=searchRegWord) | Q(reference__authors__firstName=searchWord) | - Q(reference__authors__lastName=searchWord)) + Q(tags__in=[searchRegWord]) | Q(collections__in=[searchRegWord]) | Q(reference__authors__firstName__in=[searchRegWord]) | + Q(reference__authors__lastName__in=[searchRegWord])) allFilteredSearchObjects = self.__filtersearchedPaper(filteredPaper) else: searchquery = Q() if paperTitle and paperTitle.strip(): - searchTitleQuery = Q(reference__title__icontains=paperTitle) + regSearch = '.*' + paperTitle + '.*' + searchRegWord = re.compile(regSearch, re.IGNORECASE) + searchTitleQuery = Q(reference__title=searchRegWord) searchquery = searchquery & searchTitleQuery if doi and doi.strip(): - searchdoiquery = Q(reference__DOI__icontains=doi) + regSearch = '.*' + doi + '.*' + searchRegWord = re.compile(regSearch, re.IGNORECASE) + searchdoiquery = Q(reference__DOI=searchRegWord) searchquery = searchquery & searchdoiquery - if tags and tags.strip(): - searchtagquery = Q(tags__icontains=tags) + if tags and len(tags)>0: + newtagList = [] + for item in tags: + regSearch = '.*' + item + '.*' + searchRegWord = re.compile(regSearch, re.IGNORECASE) + newtagList.append(searchRegWord) + searchtagquery = Q(tags__in=newtagList) searchquery = searchquery & searchtagquery - if len(collectionList) > 0: - searchcollectionquery = Q(collections__in=collectionList) + if collectionList and len(collectionList) > 0: + newcollectionList = [] + for item in collectionList: + regSearch = '.*' + item + '.*' + searchRegWord = re.compile(regSearch, re.IGNORECASE) + newcollectionList.append(searchRegWord) + searchcollectionquery = Q(collections__in=newcollectionList) searchquery = searchquery & searchcollectionquery - if len(authorsList) > 0: - lastnamelist = [name.split(" ")[1] for name in authorsList] - searchauthorquery = Q(reference__authors__lastName__in=lastnamelist) - searchquery = searchquery & searchauthorquery - if len(publicationList) > 0: - searchpubquery = Q(reference__journal__fullName__in=publicationList) + if authorsList and len(authorsList) > 0: + namelist = [name.split(" ") for name in authorsList] + newnamelist = [] + for sublist in namelist: + for item in sublist: + regSearch = '.*' + item + '.*' + searchRegWord = re.compile(regSearch, re.IGNORECASE) + newnamelist.append(searchRegWord) + searchlastauthorquery = Q(reference__authors__lastName__in=newnamelist) + searchfirstauthorquery = Q(reference__authors__firstName__in=newnamelist) + searchquery = searchquery & (searchlastauthorquery | searchfirstauthorquery) + if publicationList and len(publicationList) > 0: + newpubList = [] + for item in publicationList: + regSearch = '.*' + item + '.*' + searchRegWord = re.compile(regSearch, re.IGNORECASE) + newpubList.append(searchRegWord) + searchpubquery = Q(reference__journal__fullName__in=newpubList) searchquery = searchquery & searchpubquery if searchquery: filteredPaper = Paper.objects.filter(searchquery) @@ -100,6 +126,10 @@ def insertIntoPapers(self,paperdata): paper.save() return str(paper.id) + def insertDOI(self,id,doi): + """ Inserts into collection""" + paper = Paper.objects(id=id).update(info__doi=doi) + return paper def __filtersearchedPaper(self, filteredPaper): diff --git a/web/project/routes.py b/web/project/routes.py index 9d560dee..4b949ce2 100644 --- a/web/project/routes.py +++ b/web/project/routes.py @@ -81,6 +81,13 @@ def qrespcurator(): """ return render_template('qrespcurator.html') +@app.route('/startfromscratch') +def startfromscratch(): + """ clears session + """ + session.clear() + return redirect(url_for('details')) + @csrf.exempt @app.route('/uploadFile',methods=['POST','GET']) def uploadFile(): @@ -145,6 +152,7 @@ def uploadFile(): infoform = InfoForm(**paperform.data) infoform.tags.data = ", ".join(paperform.tags.data) infoform.collections.data = ", ".join(paperform.collections.data) + infoform.notebookFile.data = paperform.info.notebookFile.data session["info"] = infoform.data session["PIs"] = [form.data for form in infoform.PIs.entries] session["tags"] = [form for form in paperform.tags.data] @@ -217,15 +225,22 @@ def setproject(): absPath = pathDetails['folderAbsolutePath'] ftpclient = session.get("sftpclient") ftp = ftp_dict[ftpclient] - dtree = Dtree(ftp, absPath.rsplit("/", 1)[0], session) + if "/" in absPath: + dtree = Dtree(ftp, absPath.rsplit("/", 1)[0], session) + else: + dtree = Dtree(ftp, absPath, session) dtree.openFileToReadConfig("qresp.ini") services = dtree.fetchServices() isConfig = dtree.checkIsConfigFile() session["folderAbsolutePath"] = absPath - session["ProjectName"] = absPath.rsplit("/",1)[1] + if "/" in absPath: + session["ProjectName"] = absPath.rsplit("/",1)[1] + else: + session["ProjectName"] = absPath return jsonify(folderAbsolutePath=absPath,isConfigFile=isConfig,services=services) except Exception as e: - flash("Could not connect to server. Plase try again!" + str(e)) + print(e) + flash("Could not connect to server. Plase try again! " + str(e)) return jsonify(folderAbsolutePath=absPath, isConfigFile=isConfig, services=services) @@ -301,7 +316,7 @@ def info(): session["PIs"] = [form.data for form in infoform.PIs.entries] session["tags"] = infoform.tags.data.split(",") session["collections"] = infoform.collections.data.split(",") - session["notebookFile"] = infoform.mainnotebookfile.data + session["notebookFile"] = infoform.notebookFile.data msg = "Info Saved" return jsonify(data=msg) return jsonify(data=infoform.errors) @@ -826,9 +841,8 @@ def searchWord(): publicationList = json.loads(request.args.get('publicationList')) selectedserver = session.get("selectedserver") fetchdata = FetchDataFromAPI(selectedserver) - url = '/api/search' + "?searchWord=" + searchWord - # url = '/api/search'+"?searchWord="+searchWord+"&paperTitle="+paperTitle+"&doi="+doi+"&tags="+tags+"&collectionList="+",".join(collectionList) + \ - # "&authorsList="+",".join(authorsList)+"&publicationList="+",".join(publicationList) + url = '/api/search'+"?searchWord="+searchWord+"&paperTitle="+paperTitle+"&doi="+doi+"&tags="+tags+"&collectionList="+",".join(collectionList) + \ + "&authorsList="+",".join(authorsList)+"&publicationList="+",".join(publicationList) allpaperslist = fetchdata.fetchOutput(url) return jsonify(allpaperslist=allpaperslist) except Exception as e: @@ -844,9 +858,12 @@ def paperdetails(paperid): :return: template: paperdetails.html """ try: - dao = PaperDAO() - paperdetail = dao.getPaperDetails(paperid) - workflowdetail = dao.getWorkflowDetails(paperid) + selectedserver = session.get("selectedserver") + fetchdata = FetchDataFromAPI(selectedserver) + url = '/api/paper/'+paperid + paperdetail = fetchdata.fetchOutput(url) + url = '/api/workflow/'+paperid + workflowdetail = fetchdata.fetchOutput(url) return render_template('paperdetails.html', paperdetail=paperdetail, workflowdetail=workflowdetail) except Exception as e: print(e) @@ -870,7 +887,10 @@ def chartworkflow(): paperid = str(paperid).strip() chartid = request.args.get('chartid', 0, type=str) chartid = str(chartid).strip() - chartworkflowdetail = dao.getWorkflowForChartDetails(paperid, chartid) + selectedserver = session.get("selectedserver") + fetchdata = FetchDataFromAPI(selectedserver) + url = '/api/paper/' + paperid + '/chart/' + chartid + chartworkflowdetail = fetchdata.fetchOutput(url) return jsonify(chartworkflowdetail=chartworkflowdetail) except Exception as e: print(e) @@ -906,12 +926,31 @@ def getDescriptor(): return jsonify(content),400 return jsonify(content),200 -@app.route('/mint') +@app.route('/mint', methods=['POST','GET']) def mint(): """ Fetches the mint page """ return render_template('mint.html') +@app.route('/insertDOI', methods=['POST','GET']) +def insertDOI(): + """ Inserts DOI to database + """ + try: + paperid = request.args.get('paperId', 0, type=str) + paperid = str(paperid).strip() + doi = request.args.get('doi', 0, type=str) + doi = str(doi).strip() + dao = PaperDAO() + if paperid and doi: + dao.insertDOI(paperid,doi) + except Exception as e: + print(e) + content = {'Failed': 'Could Not Insert'} + return jsonify(content), 500 + content = {'Success': 'Inserted'} + return jsonify(content), 200 + def main(port): app.secret_key = '\xfd{H\xe5<\x95\xf9\xe3\x96.5\xd1\x01O {{ render_field_curate(infoform.collections) }} {{ render_field_curate(infoform.tags) }} - {{ render_field_curate(infoform.mainnotebookfile) }} + {{ render_field_curate(infoform.notebookFile) }} Mint D } function insertDOI(link,doi){ - $.ajax({ - type : "POST", - url : "REST/papers/insertDOI", - dataType : "json", - data : JSON.stringify({ - paperId : paperId, - doi : doi - }) - }).done(function(response) { - if (response) { - var redirect = link; - window.location.replace(redirect); - - return false; - } else { - alert("An error occurred."); - } - }).fail( - function(jqXHR, textStatus, - errorThrown) { - alert(jqXHR.responseText.message + ': ' + errorThrown); - //window.location.href = "search.jsp"; - }).always(function() { - //alert("Complete!"); - }); - } + $.getJSON('/insertDOI', { + paperId : paperId, + doi : doi + }, function (data) { + if (data) { + var redirect = link; + window.location.replace(redirect); + + return false; + } else { + alert("An error occurred."); + } + }); + } diff --git a/web/project/templates/project.html b/web/project/templates/project.html index 8e579678..a5076bc8 100644 --- a/web/project/templates/project.html +++ b/web/project/templates/project.html @@ -58,7 +58,7 @@

    - +

    diff --git a/web/project/templates/publish.html b/web/project/templates/publish.html index 888b7058..6e082bc7 100644 --- a/web/project/templates/publish.html +++ b/web/project/templates/publish.html @@ -19,7 +19,7 @@

    + style="font-weight: bold; height: 50px;" onclick="window.location='/acknowledgement';"> diff --git a/web/project/templates/qrespcurator.html b/web/project/templates/qrespcurator.html index 2f0d0536..f363f3fb 100644 --- a/web/project/templates/qrespcurator.html +++ b/web/project/templates/qrespcurator.html @@ -22,7 +22,7 @@ + style="font-weight: bold; height: 50px;" onclick="window.location='/startfromscratch';" > 0: serverList = servernames else: @@ -413,7 +417,6 @@ def __init__(self,servernames): serverList = [qrespserver['qresp_server_url'] for qrespserver in serverslist.getServersList()] global LOCALHOST - print(LOCALHOST) if LOCALHOST: serverList.append(LOCALHOST) self.__servernames = serverList @@ -423,17 +426,22 @@ def fetchOutput(self,apiname): :return: object: sends descriptor to server """ outDict = {} + output = None + self.__servernames = list(set(self.__servernames)) for snames in self.__servernames: url = snames + apiname print("url>",url) headers = {'Application': 'qresp', 'Accept': 'application/json', 'Content-Type': 'application/json'} response = requests.get(url,headers=headers, verify=False) if response.status_code == 200: - if "search" in apiname: + if "paper" or "workflow" in apiname: + output = response.json() + elif "search" in apiname: outDict.update({eachsearchObj['_Search__title']:eachsearchObj for eachsearchObj in response.json()}) + output = list(outDict.values()) else: outDict.update({eachsearchObj:eachsearchObj for eachsearchObj in response.json()}) - output = list(outDict.values()) + output = list(outDict.values()) return output diff --git a/web/project/views.py b/web/project/views.py index fdc173b1..863d509d 100644 --- a/web/project/views.py +++ b/web/project/views.py @@ -89,7 +89,7 @@ class InfoForm(Form): PIs = FieldList(FormField(NameForm), label='Principal Investigator(s)', description='Enter PI(s)', min_entries=1, validators=[validators.DataRequired()]) collections = StringField(label='PaperStack',validators = [validators.DataRequired("Please enter keywords")], description='Enter name(s) defining group of papers (e.g. according to source of fundings)',render_kw={"placeholder": "Enter collection to which project belongs"}) tags = StringField(label='Keywords',validators = [validators.DataRequired("Please enter keywords")], description='Enter Keyword(s) (e.g: DFT, organic materials, charge transfer): they facilitate paper searches using Qresp | Explorer',render_kw={"placeholder": "Enter tags for the project"}) - mainnotebookfile = StringField('Main Notebook File', description='Enter name of a notebook file, this file may serve as a table of contents and may contain links to all datasets, charts, scripts, tools and documentation. Use the Paper Content widget (on the left) to fill this field',render_kw={"placeholder": "Enter main notebook filename"}) + notebookFile = StringField('Main Notebook File', description='Enter name of a notebook file, this file may serve as a table of contents and may contain links to all datasets, charts, scripts, tools and documentation. Use the Paper Content widget (on the left) to fill this field',render_kw={"placeholder": "Enter main notebook filename"}) doi = StringField('DOI',description='DOI minted by Qresp',render_kw={'readonly': True}) class ExtraForm(Form): diff --git a/web/requirements.txt b/web/requirements.txt index fe657a5e..9d88081c 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -25,6 +25,6 @@ schedule flask-sitemap requests_oauthlib mongomock -connexion +connexion[swagger-ui] coverage nose2 \ No newline at end of file diff --git a/web/setup.py b/web/setup.py index 041e8dad..88c3e53c 100644 --- a/web/setup.py +++ b/web/setup.py @@ -36,7 +36,7 @@ 'flask-sitemap', 'requests_oauthlib', 'mongomock', - 'connexion', + 'connexion[swagger-ui]', 'coverage', 'nose2' ], From c834411b5883ad070066c87ea0a3bdfc4a6925bd Mon Sep 17 00:00:00 2001 From: atanikan Date: Fri, 11 Jan 2019 14:25:41 -0600 Subject: [PATCH 08/15] Added more tests and OS checks with travis --- .travis.yml | 5 +++++ README.md | 2 ++ web/project/api.py | 3 +++ .../tests/__pycache__/__init__.cpython-36.pyc | Bin 149 -> 0 bytes .../__pycache__/test_paperDAO.cpython-36.pyc | Bin 2022 -> 0 bytes .../__pycache__/test_routes.cpython-36.pyc | Bin 914 -> 0 bytes web/project/tests/test_paperDAO.py | 17 ++++++++++------- 7 files changed, 20 insertions(+), 7 deletions(-) delete mode 100644 web/project/tests/__pycache__/__init__.cpython-36.pyc delete mode 100644 web/project/tests/__pycache__/test_paperDAO.cpython-36.pyc delete mode 100644 web/project/tests/__pycache__/test_routes.cpython-36.pyc diff --git a/.travis.yml b/.travis.yml index 0f604c96..9d9d6faa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,11 @@ python: - "3.5" - "3.6" # command to install dependencies +os: + - linux + - osx + - windows + install: - cd web - pip install -r requirements.txt diff --git a/README.md b/README.md index 5b236c94..65a1cfe0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://travis-ci.com/atanikan/Qresp.svg?branch=qresp-1.1)](https://travis-ci.com/atanikan/Qresp) + # Qresp [Qresp](http://qresp.org) software repository. diff --git a/web/project/api.py b/web/project/api.py index 225e7be4..e40968ea 100644 --- a/web/project/api.py +++ b/web/project/api.py @@ -1,4 +1,7 @@ from .paperdao import * + +#edit swagger.yml file for method changes + def search(searchWord=None,paperTitle=None,doi=None,tags=None,collectionList=None,authorsList=None,publicationList=None): """ This function responds to a request for /api/search diff --git a/web/project/tests/__pycache__/__init__.cpython-36.pyc b/web/project/tests/__pycache__/__init__.cpython-36.pyc deleted file mode 100644 index a487519c40812ca491e14b70d00cbda4c359eeab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 149 zcmXr!<>fL8(~V&Og2x~N1{i@12OutH0TL+;!3>&=ek&P@K*9*(my@$qOlWaxQE^OS zN@j^kOh9FFMq*KJKv8~HYH~?&Okh!JaY0OZYEn!AL@1^twHPQKAD@|*SrQ+wS5SG2 T!zMRBr8Fni4rF#Q5HkP()SV^d diff --git a/web/project/tests/__pycache__/test_paperDAO.cpython-36.pyc b/web/project/tests/__pycache__/test_paperDAO.cpython-36.pyc deleted file mode 100644 index eacd236120d589fb5ed816e08d348a7f1957184f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2022 zcmbtV&2Jnv6u0MlHyfHnp}p`iQlT&@By9mzDhRdRNWH)|rKzY!D~%>=Z!(kFnZfoF zy4sw!M+$!j|B|nq_!l_wo_8}@!g8ude)c>+zmMO0w!c_kZ@>BVt9}#^@+Vnp4$L1y zmn9HJ7%fPQqg9X*jVZ;sU08?nH9NMAj1xO%bYqt}%zaH_&;0wb&pqxZ-V?$+=D#M) z=X7Sr0dKM3)Dk%-d3#1cD}7F))?Z+g^dhQ*LoQV}l^o`DXMk*M7IX^zDRhaRG6LIZ z?++4L+DXN+)ZXbCjsjN=`TI%tLH~v1LiSUZsW1BnuLdWn7#)Zz=L04C&jgos|BN5? z>p82hz_>ItsneQ^{YOuC>sQ(V&83f~(GNleU`HC?kcIq*(B)@9IEg7EvBfB}01%tm z%z+W$VIGVw^H~6+$663WTX!(zgN3;PMv2XZ-iF?VE~h{g$tk1CV%Cg;M$n1+nE%G7 zMA^BMyM}r|{n?xosMd_mK>nPn_I!0pS%941H_k1!mUpJOYX1nSeMWvDFNs_u^}2nU}{|$#J804cOZ1Rgtd)lr7T{*G^ptrGqh~(EJB=D&<)vnD*F_ z;j{-;Q9v%Us?=^Kr7R{bbZa-4RT*8=o8O}&`;WU-Szd@8bG83i+gYi!RY@?CsuLXu zo)%{2IuAN8h8{blYmv$<}hE?3xWt zz1Xk^yQ!BDCW_3+Ii1p+!e~uNZYdkHJ;S4nht8VZX;OxYWta`av?#($P-Vz}$(AzP ziyR$Xl&anYkh)C*z5_wsh756`4#gPKR{$rn#Z7QV+bpg0CTPD@j>}h3xQ_LZ{73Km zph9{;h~ohylbinnV)dT4Lgv#e$P6nH*5jihn`cxu%YcaZ1Owaxrw=YA5DgK*g4hAF zKtbGwdAtQEEa_SW!;OoOu4w!i_Ale`(MlYyY-izryokF|@%fdAKUf)YBi_RcBuVgcfK%5_l2OIR=Gek{{ArvP z^A&Lm)C$xs_K@5~(mWjV8J0qwwv OSbP>-7uLqt0sRMpLsNyA9sPR`Py zKcxSpzkyrPDSshT?<6~QfC|v*jz_-t_)af(cDld6A7w|3kUwPW2_b(AuYCtW5k(7< zwqKf3s9C{gK^jnUN>rf2TcScur{O9{BOR;gD>KJXcUFYL*-s>i?*NC466yn4*EtrO z2dG=B=a6WE@F`PNv0IV`EqEHLNX1Y`ssmu$cRyBhr^nhlo6|KAei3U3uib^<$dsSggM%V3Cc1Q>D73n`_(@C*z5A2l>+IZWV>78H?p1bnoqv&Lc4n%n z&YjIZ7;Wopp+9GJyObfAZK5!!${yC&KC;@K*L{?Nl!V?4{z*P;nvaB|_(nJyzn0qw zN~noosma^WfzorQrFma1$^ldxii0hRP5*%fWgvLN@(hwCnSw!Rvn^*^IA(HEidtUk1L#^*O6Y*zJbiR07~8|XQ4)JDL@8%l2;UWA zR;hV`d0z-r(RK_+wkH_St1a?I2D)hi2v}JRJmw(}nl;34_yqiOtg@cZ%ZWp|zOx>G n>i{3a+EBvX5X$U9@FAZ3AD!oKTB*l4b=yTw%2}W7?{WGU1$)#8 diff --git a/web/project/tests/test_paperDAO.py b/web/project/tests/test_paperDAO.py index f14a9305..b57b458d 100644 --- a/web/project/tests/test_paperDAO.py +++ b/web/project/tests/test_paperDAO.py @@ -1,15 +1,13 @@ import unittest -from unittest import TestCase from project.paperdao import * import os -import mongomock import json def warn(*args, **kwargs): pass import warnings warnings.warn = warn -class TestPaperDAO(TestCase): +class TestPaperDAO(unittest.TestCase): def setUp(self): """ Sets up database to test @@ -25,6 +23,14 @@ def setUp(self): paper = Paper(**paperdata) paper.save() + def tearDown(self): + """ + Sets up database to test + """ + paper = Paper() + paper.drop_collection() + + def test_getCollectionList(self): """ Tests if all collections exist @@ -63,7 +69,6 @@ def test_getFilteredPaperObjectsForAuthors(self): """ dao = PaperDAO() allSearchObjects = dao.getAllFilteredSearchObjects(authorsList=['marco']) - print(len(allSearchObjects)) self.assertTrue(list(allSearchObjects)) def test_getFilteredPaperObjectsForSearchWord(self): @@ -72,7 +77,6 @@ def test_getFilteredPaperObjectsForSearchWord(self): """ dao = PaperDAO() allSearchObjects = dao.getAllFilteredSearchObjects(searchWord='photo') - print(len(allSearchObjects)) self.assertTrue(list(allSearchObjects)) def test_getFilteredPaperObjectsForTitle(self): @@ -90,7 +94,6 @@ def test_getFilteredPaperObjectsForTags(self): """ dao = PaperDAO() allSearchObjects = dao.getAllFilteredSearchObjects(tags=['photo']) - print(len(allSearchObjects)) self.assertTrue(list(allSearchObjects)) def test_getFilteredPaperObjectsForCollections(self): @@ -99,7 +102,6 @@ def test_getFilteredPaperObjectsForCollections(self): """ dao = PaperDAO() allSearchObjects = dao.getAllFilteredSearchObjects(collectionList=['miccom']) - print(len(allSearchObjects)) self.assertTrue(list(allSearchObjects)) def test_insertDOI(self): @@ -109,6 +111,7 @@ def test_insertDOI(self): dao = PaperDAO() allSearchObjects = dao.getAllFilteredSearchObjects() paper = dao.insertDOI(allSearchObjects[0]['_Search__id'],'123') + print(dao.getAllPapers()) self.assertEquals(1, paper) From 56999481ddc2dab3a16658a3b76964205d9fd90f Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 16 Jan 2019 13:58:03 -0600 Subject: [PATCH 09/15] Updates for coverall --- .travis.yml | 6 ++++-- README.md | 3 ++- web/project/tests/test_paperDAO.py | 1 - web/requirements.txt | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9d9d6faa..a4a49447 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ python: os: - linux - osx - - windows install: - cd web @@ -15,4 +14,7 @@ install: - python setup.py install # command to run tests script: - - nose2 + - nose2 -v + +after_success: + - coveralls \ No newline at end of file diff --git a/README.md b/README.md index 65a1cfe0..3f5cb6c6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -[![Build Status](https://travis-ci.com/atanikan/Qresp.svg?branch=qresp-1.1)](https://travis-ci.com/atanikan/Qresp) +[![Build Status](https://travis-ci.org/qresp-code-development/qresp.svg?branch=master)](https://travis-ci.org/qresp-code-development/qresp) +[![Coverage Status](https://coveralls.io/repos/github/qresp-code-development/qresp/badge.svg?branch=master)](https://coveralls.io/github/qresp-code-development/qresp?branch=qresp-1.1) # Qresp [Qresp](http://qresp.org) software repository. diff --git a/web/project/tests/test_paperDAO.py b/web/project/tests/test_paperDAO.py index b57b458d..e56e24ee 100644 --- a/web/project/tests/test_paperDAO.py +++ b/web/project/tests/test_paperDAO.py @@ -111,7 +111,6 @@ def test_insertDOI(self): dao = PaperDAO() allSearchObjects = dao.getAllFilteredSearchObjects() paper = dao.insertDOI(allSearchObjects[0]['_Search__id'],'123') - print(dao.getAllPapers()) self.assertEquals(1, paper) diff --git a/web/requirements.txt b/web/requirements.txt index 9d88081c..2a896d94 100644 --- a/web/requirements.txt +++ b/web/requirements.txt @@ -27,4 +27,5 @@ requests_oauthlib mongomock connexion[swagger-ui] coverage -nose2 \ No newline at end of file +nose2 +python-coveralls \ No newline at end of file From 2152d9e6aa6758b885de102129a2656daee83dcd Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 16 Jan 2019 14:14:25 -0600 Subject: [PATCH 10/15] Updates for coverall --- .travis.yml | 36 ++++++++++++++++++++++++++++-------- README.md | 4 ++-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4a49447..45adebac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,32 @@ language: python -python: - - "3.4" - - "3.5" - - "3.6" +#python: +# - "3.4" +# - "3.5" +# - "3.6" # command to install dependencies -os: - - linux - - osx +#os: +# - linux +# - osx +matrix: + include: + - os: linux + python: 3.4 + env: TOXENV=py34 + - os: linux + python: 3.5 + env: TOXENV=py35 + - os: linux + python: 3.6 + env: TOXENV=py36 + - os: osx + python: 3.4 + env: TOXENV=py34 + - os: osx + python: 3.5 + env: TOXENV=py35 + - os: osx + python: 3.6 + env: TOXENV=py36 install: - cd web @@ -14,7 +34,7 @@ install: - python setup.py install # command to run tests script: - - nose2 -v + - nose2 --with-coverage -v after_success: - coveralls \ No newline at end of file diff --git a/README.md b/README.md index 3f5cb6c6..7edce470 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![Build Status](https://travis-ci.org/qresp-code-development/qresp.svg?branch=master)](https://travis-ci.org/qresp-code-development/qresp) -[![Coverage Status](https://coveralls.io/repos/github/qresp-code-development/qresp/badge.svg?branch=master)](https://coveralls.io/github/qresp-code-development/qresp?branch=qresp-1.1) - +[![Coverage Status](https://coveralls.io/repos/github/qresp-code-development/qresp/badge.svg?branch=qresp-1.1)](https://coveralls.io/github/qresp-code-development/qresp?branch=qresp-1.1) +[![Coverage Status](https://coveralls.io/repos/github/qresp-code-development/qresp/badge.svg?branch=master)](https://coveralls.io/github/qresp-code-development/qresp?branch=master) # Qresp [Qresp](http://qresp.org) software repository. From 493b22057b4c38d6b16871a28a0749bec60db8de Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 16 Jan 2019 14:24:22 -0600 Subject: [PATCH 11/15] Updates for coverall --- .travis.yml | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 45adebac..128a47f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,14 +19,34 @@ matrix: python: 3.6 env: TOXENV=py36 - os: osx - python: 3.4 - env: TOXENV=py34 - - os: osx - python: 3.5 - env: TOXENV=py35 + env: PYTHON=3.4.4 + language: generic - os: osx - python: 3.6 - env: TOXENV=py36 + language: generic + env: PYTHON=3.5.1 + +before_install: | + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + brew update + # Per the `pyenv homebrew recommendations `_. + brew install openssl readline + # See https://docs.travis-ci.com/user/osx-ci-environment/#A-note-on-upgrading-packages. + # I didn't do this above because it works and I'm lazy. + brew outdated pyenv || brew upgrade pyenv + # virtualenv doesn't work without pyenv knowledge. venv in Python 3.3 + # doesn't provide Pip by default. So, use `pyenv-virtualenv `_. + brew install pyenv-virtualenv + pyenv install $PYTHON + # I would expect something like ``pyenv init; pyenv local $PYTHON`` or + # ``pyenv shell $PYTHON`` would work, but ``pyenv init`` doesn't seem to + # modify the Bash environment. ??? So, I hand-set the variables instead. + export PYENV_VERSION=$PYTHON + export PATH="/Users/travis/.pyenv/shims:${PATH}" + pyenv-virtualenv venv + source venv/bin/activate + # A manual check that the correct version of Python is running. + python --version + fi install: - cd web From 7b034097602c772ddcfa288547db659e258373a0 Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 16 Jan 2019 14:33:36 -0600 Subject: [PATCH 12/15] Updates for coverall with Readme and travis for OSX --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7edce470..fddcc6f3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ [![Build Status](https://travis-ci.org/qresp-code-development/qresp.svg?branch=master)](https://travis-ci.org/qresp-code-development/qresp) -[![Coverage Status](https://coveralls.io/repos/github/qresp-code-development/qresp/badge.svg?branch=qresp-1.1)](https://coveralls.io/github/qresp-code-development/qresp?branch=qresp-1.1) [![Coverage Status](https://coveralls.io/repos/github/qresp-code-development/qresp/badge.svg?branch=master)](https://coveralls.io/github/qresp-code-development/qresp?branch=master) # Qresp [Qresp](http://qresp.org) software repository. From 3d86af609f32a60388dc40aa1a23902ee719d6d9 Mon Sep 17 00:00:00 2001 From: atanikan Date: Wed, 23 Jan 2019 19:35:52 -0600 Subject: [PATCH 13/15] Added support for coverall --- .travis.yml | 9 +-------- docker-compose.override.yml | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 128a47f7..bcf4edc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,4 @@ language: python -#python: -# - "3.4" -# - "3.5" -# - "3.6" -# command to install dependencies -#os: -# - linux -# - osx matrix: include: - os: linux @@ -45,6 +37,7 @@ before_install: | pyenv-virtualenv venv source venv/bin/activate # A manual check that the correct version of Python is running. + pip install --upgrade setuptools python --version fi diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 891190b0..eca42112 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -16,4 +16,4 @@ services: - MAIL_PWD= volumes: - ./web/:/usr/src/app/web - command: flask run --host=0.0.0.0 --port 8080 + command: python run.py --host=0.0.0.0 --port 8080 From c7badf9ca1b7e210ef8fb1579abebb5a5fb61a22 Mon Sep 17 00:00:00 2001 From: atanikan Date: Thu, 24 Jan 2019 11:46:28 -0600 Subject: [PATCH 14/15] Added more unit tests and corrections based on unit tests --- .travis.yml | 3 - web/project/__init__.py | 15 ++-- web/project/api.py | 6 +- web/project/routes.py | 120 ++++++++++++++++------------- web/project/tests/test_paperDAO.py | 45 ++++++++--- web/project/tests/test_routes.py | 86 ++++++++++++++++++--- web/project/util.py | 4 +- web/run.py | 2 +- 8 files changed, 191 insertions(+), 90 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcf4edc0..56e36c69 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,6 @@ matrix: - os: osx env: PYTHON=3.4.4 language: generic - - os: osx - language: generic - env: PYTHON=3.5.1 before_install: | if [ "$TRAVIS_OS_NAME" == "osx" ]; then diff --git a/web/project/__init__.py b/web/project/__init__.py index b843c590..d65b3197 100644 --- a/web/project/__init__.py +++ b/web/project/__init__.py @@ -22,8 +22,6 @@ MONGODB_PASSWORD = os.environ.get("MONGODB_PASSWORD") MONGODB_DB = os.environ.get("MONGODB_DB") - -#app = Flask(__name__) # Create the application instance connexionapp = connexion.App(__name__, specification_dir='./') @@ -31,6 +29,7 @@ connexionapp.add_api('swagger.yml') app = connexionapp.app app.secret_key = '\\\xfcS\x1e\x8f\xfb]6\x1e.\xa8\xb3\xe1x\xc8\x8e\xc1\xeb5^x\x81\xcc\xd5' + csrf = CSRFProtect(app) SESSION_TYPE = 'filesystem' app.config.from_object(__name__) @@ -45,12 +44,14 @@ os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' app.config['MAIL_ADDR'] = MAIL_ADDR app.config['MAIL_PWD'] = MAIL_PWD -app.config['MONGODB_HOST'] = 'paperstack.uchicago.edu' -app.config['MONGODB_PORT'] = 27017 -app.config['MONGODB_USERNAME'] = 'qresp_user_explorer' -app.config['MONGODB_PASSWORD'] = 'qresp_pwd' -app.config['MONGODB_DB'] = 'explorer' +app.config['MONGODB_HOST'] = MONGODB_HOST +app.config['MONGODB_PORT'] = MONGODB_PORT +app.config['MONGODB_USERNAME'] = MONGODB_USERNAME +app.config['MONGODB_PASSWORD'] = MONGODB_PASSWORD +app.config['MONGODB_DB'] = MONGODB_DB + Session(app) ext = Sitemap(app) + from project import routes diff --git a/web/project/api.py b/web/project/api.py index e40968ea..12b6a5bf 100644 --- a/web/project/api.py +++ b/web/project/api.py @@ -84,7 +84,7 @@ def paper(id): except Exception as e: msg = "Exception in paper api " + str(e) print("Exception in paper api ", e) - return e, 500 + return e, 400 return paperdetail def workflow(id): @@ -101,7 +101,7 @@ def workflow(id): except Exception as e: msg = "Exception in workflow api " + str(e) print("Exception in workflow api ", e) - return msg,500 + return msg,400 return workflowdetail def chart(id,cid): @@ -118,5 +118,5 @@ def chart(id,cid): except Exception as e: msg = "Exception in chart api " + str(e) print("Exception in chart api ", e) - return msg,500 + return msg,400 return chartworkflowdetail \ No newline at end of file diff --git a/web/project/routes.py b/web/project/routes.py index 4b949ce2..37f0314a 100644 --- a/web/project/routes.py +++ b/web/project/routes.py @@ -12,7 +12,7 @@ from flask_cors import CORS -from .paperdao import * +from .paperdao import PaperDAO,MongoDBConnection from .util import * from .views import * from project import app @@ -60,7 +60,7 @@ def index(): """ Fetches the homepage """ #TO BE DELETED - SetLocalHost(str(request.host_url).strip("/")) + #SetLocalHost(str(request.host_url).strip("/")) return render_template('index.html') @app.route('/downloads/') @@ -158,11 +158,12 @@ def uploadFile(): session["tags"] = [form for form in paperform.tags.data] session["collections"] = [form for form in paperform.collections.data] session["workflow"] = paperform.workflow.data - return jsonify(out="Processed text") - except: - return jsonify(error="Could not process json file") - + return jsonify(out="Processed text"), 200 + except Exception as e: + print(e) + return jsonify(error=str(e)), 400 +@csrf.exempt @app.route('/details', methods=['POST', 'GET']) def details(): """ This method helps in storing the details of the person using the curator. @@ -193,12 +194,12 @@ def server(): return redirect(url_for('project')) except Exception as e: flash("Could not connect to server. Plase try again!"+str(e)) - render_template('server.html', form=form) - return render_template('server.html', form=form) + render_template('server.html', form=form), 400 + return render_template('server.html', form=form), 200 @csrf.exempt -@app.route('/project', methods=['POST', 'GET']) +@app.route('/project', methods=['GET']) def project(): """ This method helps in populating the project form. """ @@ -237,11 +238,11 @@ def setproject(): session["ProjectName"] = absPath.rsplit("/",1)[1] else: session["ProjectName"] = absPath - return jsonify(folderAbsolutePath=absPath,isConfigFile=isConfig,services=services) + return jsonify(folderAbsolutePath=absPath,isConfigFile=isConfig,services=services), 200 except Exception as e: print(e) flash("Could not connect to server. Plase try again! " + str(e)) - return jsonify(folderAbsolutePath=absPath, isConfigFile=isConfig, services=services) + return jsonify(folderAbsolutePath=absPath, isConfigFile=isConfig, services=services), 401 @csrf.exempt @@ -252,19 +253,20 @@ def getTreeInfo(): pathDetails = request.get_json() listObjects = [] services = [] - ftpclient = session.get("sftpclient") - ftp = ftp_dict[ftpclient] - if 'folderAbsolutePath' in pathDetails: - content_path = pathDetails['folderAbsolutePath'] - else: - content_path = session.get("folderAbsolutePath") try: + ftpclient = session.get("sftpclient") + ftp = ftp_dict[ftpclient] + if 'folderAbsolutePath' in pathDetails: + content_path = pathDetails['folderAbsolutePath'] + else: + content_path = session.get("folderAbsolutePath") dtree = Dtree(ftp,content_path,session) listObjects = dtree.fetchForTree() services = dtree.fetchServices() except Exception as e: + jsonify({'errors':str(e)}), 400 print(e) - return jsonify({'listObjects': listObjects, 'services': services}) + return jsonify({'listObjects': listObjects, 'services': services}), 200 @app.route('/curate', methods=['POST', 'GET']) def curate(): @@ -318,8 +320,8 @@ def info(): session["collections"] = infoform.collections.data.split(",") session["notebookFile"] = infoform.notebookFile.data msg = "Info Saved" - return jsonify(data=msg) - return jsonify(data=infoform.errors) + return jsonify(data=msg), 200 + return jsonify(data=infoform.errors), 400 @csrf.exempt @app.route('/charts', methods=['POST', 'GET']) @@ -343,8 +345,8 @@ def charts(): chartList = [item for item in chartList if item['saveas'] not in chartform.saveas.data] chartList.append(chartform.data) session["charts"] = deepcopy(chartList) - return jsonify(chartList = chartList) - return jsonify(data=chartform.errors) + return jsonify(chartList = chartList), 200 + return jsonify(data=chartform.errors), 200 @csrf.exempt @app.route('/tools', methods=['POST', 'GET']) @@ -368,8 +370,8 @@ def tools(): toolList = [item for item in toolList if item['saveas'] not in toolform.saveas.data] toolList.append(toolform.data) session["tools"] = deepcopy(toolList) - return jsonify(toolList = toolList) - return jsonify(errors=toolform.errors) + return jsonify(toolList = toolList), 200 + return jsonify(errors=toolform.errors), 200 @csrf.exempt @app.route('/datasets', methods=['POST', 'GET']) @@ -393,8 +395,8 @@ def datasets(): datasetList = [item for item in datasetList if item['saveas'] not in datasetform.saveas.data] datasetList.append(datasetform.data) session["datasets"] = deepcopy(datasetList) - return jsonify(datasetList = datasetList) - return jsonify(data=datasetform.errors) + return jsonify(datasetList = datasetList), 200 + return jsonify(data=datasetform.errors), 200 @csrf.exempt @app.route('/scripts', methods=['POST', 'GET']) @@ -418,8 +420,8 @@ def scripts(): scriptList = [item for item in scriptList if item['saveas'] not in scriptform.saveas.data] scriptList.append(scriptform.data) session["scripts"] = deepcopy(scriptList) - return jsonify(scriptList = scriptList) - return jsonify(data=scriptform.errors) + return jsonify(scriptList = scriptList), 200 + return jsonify(data=scriptform.errors), 200 @csrf.exempt @app.route('/reference', methods=['POST', 'GET']) @@ -430,8 +432,8 @@ def reference(): if request.method == 'POST' and referenceform.validate(): session["reference"] = referenceform.data msg = "Reference Saved" - return jsonify(data=msg) - return jsonify(data=referenceform.errors) + return jsonify(data=msg), 200 + return jsonify(data=referenceform.errors), 200 @csrf.exempt @app.route('/fetchReferenceDOI', methods=['POST', 'GET']) @@ -444,8 +446,8 @@ def fetchReferenceDOI(): refdata = fetchDOI.fetchFromDOI() except Exception as e: print(e) - return jsonify(errors="Recheck your DOI") - return jsonify(fetchDOI=refdata.data) + return jsonify(errors="Recheck your DOI"), 400 + return jsonify(fetchDOI=refdata.data), 200 @csrf.exempt @app.route('/documentation', methods=['POST', 'GET']) @@ -456,8 +458,8 @@ def documentation(): if request.method == 'POST' and docform.validate(): session["documentation"] = docform.data msg = "Documentation Saved" - return jsonify(data=msg) - return jsonify(data=docform.errors) + return jsonify(data=msg), 200 + return jsonify(data=docform.errors), 200 @csrf.exempt @app.route('/workflow', methods=['POST', 'GET']) @@ -559,10 +561,9 @@ def workflow(): session["workflow"] = workflowinfo.data except Exception as e: print(e) - return jsonify({'workflow': workflow.__dict__}) + return jsonify({'workflow': workflow.__dict__}), 200 -# Fetches list of qresp servers for the explorer @csrf.exempt @app.route('/publish', methods=['POST', 'GET']) def publish(): @@ -720,7 +721,7 @@ def download(): with open("papers/"+projectName+"/"+"data.json", "w") as outfile: json.dump(paperdata, outfile, default=lambda o: o.__dict__, separators=(',', ':'), sort_keys=True, indent=3) session["paper"] = paperdata - return jsonify(paper=paperdata) + return jsonify(paper=paperdata), 200 @csrf.exempt @@ -737,15 +738,13 @@ def acknowledgement(): def qrespexplorer(): """ Fetches the explorer homepage """ - dao = PaperDAO() - dao.getDB() form = QrespServerForm() serverslist = Servers() form.serverList = [qrespserver['qresp_server_url'] for qrespserver in serverslist.getServersList()] if request.method == 'POST': - if request.form.get('serversList'): - session["selectedserver"] = request.form.get('serversList') + if request.form.getlist('serversList'): + session["selectedserver"] = request.form.getlist('serversList') else: session['selectedserver'] = form.serverList return redirect(url_for('search')) @@ -760,8 +759,8 @@ def verifypasscode(): confirmpasscode = app.config['passkey'] if request.method == 'POST' and form.validate(): if confirmpasscode == form.passcode.data: - return jsonify(msg="success") - return jsonify(msg="failed") + return jsonify(msg="success"),200 + return jsonify(msg="failed"),401 # Fetches list of qresp admin @csrf.exempt @@ -807,14 +806,23 @@ def search(): :return: template: search.html """ allpaperslist = [] + collectionlist = [] + authorslist = [] + publicationlist = [] + allPapersSize = 0 try: selectedserver = session.get("selectedserver") fetchdata = FetchDataFromAPI(selectedserver) - allpaperslist = fetchdata.fetchOutput('/api/search') - collectionlist = fetchdata.fetchOutput('/api/collections') - authorslist = fetchdata.fetchOutput('/api/authors') - publicationlist = fetchdata.fetchOutput('/api/publications') - allPapersSize = len(allpaperslist) + fetchallpaperslist = fetchdata.fetchOutput('/api/search') + fetchcollectionlist = fetchdata.fetchOutput('/api/collections') + fetchauthorslist = fetchdata.fetchOutput('/api/authors') + fetchpublicationlist = fetchdata.fetchOutput('/api/publications') + if fetchallpaperslist and len(fetchallpaperslist)>0: + allpaperslist = fetchallpaperslist + collectionlist = fetchcollectionlist + authorslist = fetchauthorslist + publicationlist = fetchpublicationlist + allPapersSize = len(allpaperslist) except Exception as e: print(e) flash('Error in search. ' + str(e)) @@ -830,6 +838,7 @@ def searchWord(): """ filtering paper content :return: object: allpaperlist """ + allpaperslist = [] try: dao = PaperDAO() searchWord = request.args.get('searchWord', type=str) @@ -844,10 +853,11 @@ def searchWord(): url = '/api/search'+"?searchWord="+searchWord+"&paperTitle="+paperTitle+"&doi="+doi+"&tags="+tags+"&collectionList="+",".join(collectionList) + \ "&authorsList="+",".join(authorsList)+"&publicationList="+",".join(publicationList) allpaperslist = fetchdata.fetchOutput(url) - return jsonify(allpaperslist=allpaperslist) + return jsonify(allpaperslist=allpaperslist), 200 except Exception as e: print(e) flash('Error in search. ' + str(e)) + return jsonify(allpaperslist=allpaperslist), 400 # Fetches details of chart based on user click @@ -857,6 +867,8 @@ def paperdetails(paperid): """ fetching papers details content :return: template: paperdetails.html """ + paperdetail = [] + workflowdetail = [] try: selectedserver = session.get("selectedserver") fetchdata = FetchDataFromAPI(selectedserver) @@ -868,8 +880,6 @@ def paperdetails(paperid): except Exception as e: print(e) flash('Error in paperdetails. ' + str(e)) - paperdetail = [] - workflowdetail = [] return render_template('paperdetails.html', paperdetail=paperdetail, workflowdetail=workflowdetail) @@ -881,6 +891,7 @@ def chartworkflow(): """ Fetching chart workflow content :return: object: chartworkflow """ + chartworkflowdetail = [] try: dao = PaperDAO() paperid = request.args.get('paperid', 0, type=str) @@ -891,10 +902,11 @@ def chartworkflow(): fetchdata = FetchDataFromAPI(selectedserver) url = '/api/paper/' + paperid + '/chart/' + chartid chartworkflowdetail = fetchdata.fetchOutput(url) - return jsonify(chartworkflowdetail=chartworkflowdetail) + return jsonify(chartworkflowdetail=chartworkflowdetail), 200 except Exception as e: print(e) - flash('Error in paperdetails. ' + str(e)) + flash('Error in chartdetails. ' + str(e)) + return jsonify(chartworkflowdetail=chartworkflowdetail), 400 @csrf.exempt @app.route('/getDescriptor', methods=['POST','GET']) @@ -947,7 +959,7 @@ def insertDOI(): except Exception as e: print(e) content = {'Failed': 'Could Not Insert'} - return jsonify(content), 500 + return jsonify(content), 400 content = {'Success': 'Inserted'} return jsonify(content), 200 diff --git a/web/project/tests/test_paperDAO.py b/web/project/tests/test_paperDAO.py index e56e24ee..8ef9ce81 100644 --- a/web/project/tests/test_paperDAO.py +++ b/web/project/tests/test_paperDAO.py @@ -1,5 +1,5 @@ import unittest -from project.paperdao import * +from project.paperdao import PaperDAO,MongoDBConnection,Paper import os import json def warn(*args, **kwargs): @@ -85,7 +85,6 @@ def test_getFilteredPaperObjectsForTitle(self): """ dao = PaperDAO() allSearchObjects = dao.getAllFilteredSearchObjects(paperTitle='photo') - print(len(allSearchObjects)) self.assertTrue(list(allSearchObjects)) def test_getFilteredPaperObjectsForTags(self): @@ -106,7 +105,7 @@ def test_getFilteredPaperObjectsForCollections(self): def test_insertDOI(self): """ - Tests for isertion of DOI + Tests for insertion of DOI """ dao = PaperDAO() allSearchObjects = dao.getAllFilteredSearchObjects() @@ -114,14 +113,38 @@ def test_insertDOI(self): self.assertEquals(1, paper) - # def test_getPaperDetails(self): - # self.fail() - # - # def test_getWorkflowDetails(self): - # self.fail() - # - # def test_getWorkflowForChartDetails(self): - # self.fail() + def test_getPaperDetails(self): + """ + Tests Paper details given paper id + """ + dao = PaperDAO() + allSearchObjects = dao.getAllFilteredSearchObjects() + paperDetails = dao.getPaperDetails(allSearchObjects[0]['_Search__id']) + self.assertEquals(allSearchObjects[0]['_Search__id'], paperDetails['_PaperDetails__id']) + + + def test_getWorkflowDetails(self): + """ + Tests workflow details given paper id + """ + dao = PaperDAO() + allSearchObjects = dao.getAllFilteredSearchObjects() + workflowdetails = dao.getWorkflowDetails(allSearchObjects[0]['_Search__id']) + self.assertEquals(workflowdetails['paperTitle'],allSearchObjects[0]['_Search__title']) + + + + def test_getWorkflowForChartDetails(self): + """ + Tests workflow details given chart id and paper id + :return: + """ + dao = PaperDAO() + allSearchObjects = dao.getAllFilteredSearchObjects() + paperDetails = dao.getPaperDetails(allSearchObjects[0]['_Search__id']) + chartid = paperDetails['_PaperDetails__charts'][0].id + workflowchartdetails = dao.getWorkflowForChartDetails(paperDetails['_PaperDetails__id'],chartid) + self.assertEquals(workflowchartdetails['paperTitle'],allSearchObjects[0]['_Search__title']) if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/web/project/tests/test_routes.py b/web/project/tests/test_routes.py index c0e08f65..7f86f909 100644 --- a/web/project/tests/test_routes.py +++ b/web/project/tests/test_routes.py @@ -1,7 +1,10 @@ import os import unittest - +import json from project import app +from project.views import * +from flask import Flask, request, jsonify + class RouteTests(unittest.TestCase): ############################ @@ -10,19 +13,84 @@ class RouteTests(unittest.TestCase): # executed prior to each test def setUp(self): - self.app = app.test_client() + + self.app = self.create_app() + self.client = app.test_client() + self.ctx = self.app.test_request_context() + self.ctx.push() + + self.pagenames = ['/','qrespcurator','startfromscratch','details','server','project','curate','workflow','publish', + 'acknowledgement','qrespexplorer','admin','search', + 'searchWord?searchWord=&paperTitle=&tags=&doi=&collectionList=[]&authorsList=[]&publicationList=[]', + 'paperdetails/5941869f1bd40fd44db0024a','chartworkflow?paperid=5941869f1bd40fd44db0024a&chartid=c0'] + + postpagedicts = {'details':{'firstName':'John','middleName':'L','lastName':'Doe'}, + 'server':{'serverName':'testServer','username':'testuser','password':'testpassword','isDUOAuth':'No'}, + 'charts':{'caption':'cc','number':'1','files':'a,b','imageFile':'a.png','properties':'1,2','saveas':'c0'}, + 'tools':{'packageName':'West','version':'v0','facilityName':'Xray','measurement':'xray','saveas':'t0'}, + 'datasets':{'files':'a,b','readme':'readdata','saveas':'d0'}, + 'scripts':{'files':'a,b','readme':'readdata','saveas':'s0'}, + 'reference':{'DOI':'doi/10.123','title':'Test','page':'1','publishedAbstract':'Abs','volume':'1','year':2019}, + 'fetchReferenceDOI':{}, + 'documentation':{'readme':'read'}, + 'publish':{}, + 'download':{}} + + nameform = NameForm() + nameform.firstName.data = 'John' + nameform.middleName.data = 'L.' + nameform.lastName.data = 'Doe' + detailsform = DetailsForm(**postpagedicts['details']) + serverform = ServerForm(**postpagedicts['server']) + chartform = ChartForm(**postpagedicts['charts']) + chartform.extraFields.entries = [] + toolform = ToolForm(**postpagedicts['tools']) + toolform.extraFields.entries = [] + datasetform = DatasetForm(**postpagedicts['datasets']) + datasetform.extraFields.entries = [] + scriptform = ScriptForm(**postpagedicts['scripts']) + scriptform.extraFields.entries = [] + referenceform = ReferenceForm(**postpagedicts['reference']) + referenceform.authors.entries = [] + referenceform.journal.abbrevName.data = 'JAbbr' + referenceform.journal.fullName.data = 'JFull' + documentationform = DocumentationForm(**postpagedicts['documentation']) + self.postPageForms = {'details':detailsform,'server':serverform,'charts':chartform,'tools':toolform, + 'datasets':datasetform,'scripts':scriptform,'fetchReferenceDOI':{'doi':'10.1021/nl903868w'}, + 'documentation':documentationform,'workflow':[['to','do'],['to','do']], + 'publish':{},'download':{}} # executed after each test + def create_app(self): + app = Flask(__name__) + app.secret_key = "secret" + return app + def tearDown(self): - pass + self.ctx.pop() + + def request(self,*args, **kwargs): + return self.app.test_request_context(*args, **kwargs) + + def test_getMethods(self): + for pagename in self.pagenames: + response = self.client.get(pagename, follow_redirects=True) + self.assertEquals(response.status_code,200) + + + def test_postMethods(self): + for pagename in self.postPageForms.keys(): + form = self.postPageForms[pagename] + if 'fetchReferenceDOI' in pagename or 'workflow' in pagename or 'publish' in pagename or 'download' in pagename: + response = self.client.post(pagename, data=json.dumps(form), content_type='application/json') + else: + response = self.client.post(pagename,data=form.data) + if 'details' in pagename: + self.assertEqual(response.status_code, 302) + else: + self.assertEqual(response.status_code, 200) - def test_main_page(self): - response = self.app.get('/', follow_redirects=True) - self.assertEqual(response.status_code, 200) - def test_main_page(self): - response = self.app.get('/', follow_redirects=True) - self.assertEqual(response.status_code, 200) if __name__ == "__main__": unittest.main() diff --git a/web/project/util.py b/web/project/util.py index d235b15a..cbb2f4bf 100644 --- a/web/project/util.py +++ b/web/project/util.py @@ -409,7 +409,8 @@ class FetchDataFromAPI(): """ Fetches data for search,paper details and chart details for explorer """ def __init__(self,servernames): - serverList = [] + if isinstance(servernames,str): + servernames = [servernames] if servernames and len(servernames)>0: serverList = servernames else: @@ -430,7 +431,6 @@ def fetchOutput(self,apiname): self.__servernames = list(set(self.__servernames)) for snames in self.__servernames: url = snames + apiname - print("url>",url) headers = {'Application': 'qresp', 'Accept': 'application/json', 'Content-Type': 'application/json'} response = requests.get(url,headers=headers, verify=False) if response.status_code == 200: diff --git a/web/run.py b/web/run.py index a5b9fff3..32ac3525 100644 --- a/web/run.py +++ b/web/run.py @@ -7,5 +7,5 @@ #app.run() if __name__ == "__main__": - connexionapp.run(port=80,debug=True) + connexionapp.run(host='0.0.0.0',port=80,debug=True) From 2ecdb7c64f4fd3c09b3d1d38a377eb4f5e3a7bcd Mon Sep 17 00:00:00 2001 From: atanikan Date: Fri, 25 Jan 2019 10:07:14 -0600 Subject: [PATCH 15/15] Added new docker compose for services --- docker-compose.yml.services | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 docker-compose.yml.services diff --git a/docker-compose.yml.services b/docker-compose.yml.services new file mode 100644 index 00000000..ae62d4bf --- /dev/null +++ b/docker-compose.yml.services @@ -0,0 +1,45 @@ +version: '2' + +services: + web: + restart: always + build: ./web + expose: + - 8080 + ports: + - "8080:8080" + volumes: + - ./web/:/usr/src/app/web + environment: + - KEY= + - GOOGLE_CLIENT_ID= + - GOOGLE_CLIENT_SECRET= + - REDIRECT_URI= + - MAIL_ADDR= + - MAIL_PWD= + command: python run.py --host=0.0.0.0 --port 8080 + + nginx: + restart: always + build: ./nginx + ports: + - "8001:8001" + volumes: + - /www/static + - ./papercollection:/usr/src/files + volumes_from: + - web + depends_on: + - web + + mongodb: + image: mongo:latest + container_name: "mongodb" + environment: + - MONGO_DATA_DIR=/usr/data/db + - MONGO_LOG_DIR=/dev/null + volumes: + - ./data/db:/usr/data/db + ports: + - 27017:27017 + command: mongod --smallfiles --logpath=/dev/null # --quiet