Merge pull request #113 from dod-advana/UOT-139890

feature/UOT-139890

gamechangerml/api/fastapi/routers/controls.py

@@ -24,9 +24,11 @@
     handle_sent_evals,
 )
 from gamechangerml import MODEL_PATH
+from gamechangerml.src.utilities import gc_web_api
 
 router = APIRouter()
 MODELS = ModelLoader()
+gcClient = gc_web_api.GCWebClient()
 ## Get Methods ##
 
 pipeline = Pipeline()
@@ -969,4 +971,23 @@ async def stop_process(thread_dict: dict, response: Response):
         thread_id=thread_id,
     )
 
-    return {"stopped": thread_id}
+    return {'stopped':thread_id}
+
+@router.post("/getUserData")
+async def stop_process(date_dict: dict, response: Response):
+    """Get user aggregation data for selected date and write to data folder
+    Args:
+        date_dict: dict; {startDate,endDate}
+        Response: Response class; for status codes(apart of fastapi do not need to pass param)
+    Returns:
+        Stopped thread id
+    """
+
+    data = json.loads(gcClient.getUserAggregations(start_date=date_dict['startDate'],end_date=date_dict['endDate']))
+    GC_USER_DATA = os.path.join(
+        DATA_PATH, "user_data", "UserAggregations.json"
+    )
+    with open(GC_USER_DATA,'w') as f:
+        json.dump(data,f)
+
+    return 'wrote user data to file'

gamechangerml/data/user_data/UserAggregations.json

@@ -0,0 +1 @@
+{"users": [{"idvisitor": "031A24EEF605BC74", "docs_opened": 0, "searches_made": 2, "opened": [], "export": [], "favorite": []}, {"idvisitor": "1CEAB3D67A91BFA1", "docs_opened": 34, "searches_made": 56, "opened": ["S.Res. 536 IS 117th.pdf", "MIL-D-23222 Revision A Notice 1", "OPNAVINST 3100.8B.pdf", "MIL-C-2683 Revision A Notice 4", "H.R. 5467 IH 117th.pdf"], "export": [], "favorite": ["MIL-D-23222 Revision A Notice 1 - Validation 2019-06-25.pdf", "OPNAVINST 3100.8B.pdf", "DoDD 5136.01 CH 1.pdf"]}, {"idvisitor": "23BCE26849B77287", "docs_opened": 5, "searches_made": 1, "opened": ["DoD Data Strategy.PD[\u2026]geNumber=0"], "export": [], "favorite": []}, {"idvisitor": "58751A4353CBF7C0", "docs_opened": 109, "searches_made": 1087, "opened": ["ARMY DIR 2020-06.pdf", "18-006-  .pdf", "MIL-DTL-32541 Base Document 2015-11-30.pdf", "DoDI 1235.12 CH 1.pdf", "6200.01  .pdf"], "export": ["DoDD 5137.02.pdf", "CJCSI 1805.01C.pdf", "DoDI 1215.21 CH 1.pdf", "DoDI 1322.26.pdf", "AI 40.pdf"], "favorite": ["TM 4-48.09.pdf", "STP 9-94K13-SM-TG.pdf", "ATP 3-04.19.pdf", "DoDD 3025.13 CH 1.pdf", "SECNAVINST 5239.24.pdf"]}, {"idvisitor": "60592F05B49DCD68", "docs_opened": 4, "searches_made": 24, "opened": ["DHA Procedural Instructions 6010.01.pdf", "CFR-2021-title26-vol18.pdf"], "export": [], "favorite": ["PGI 239.76 - CLOUD COMPUTING.pdf"]}, {"idvisitor": "62FC9F3443EC3AE3", "docs_opened": 33, "searches_made": 92, "opened": ["Title 9", "DFARS SUBPART 222.74", "PGI 222.74", "SECNAVINST 5800.15A.pdf", "SECNAVINST 3300.2C.pdf"], "export": [], "favorite": ["SECNAVINST 5239.24.pdf", "COMNAVRESFORCOMINST 5726.3.pdf", "OPNAVINST 5420.106A.pdf"]}, {"idvisitor": "64DC6ABB20F9B2C3", "docs_opened": 1, "searches_made": 5, "opened": ["H.R. 5467 IH 117th.pdf"], "export": [], "favorite": ["SECNAVINST 5239.24.pdf", "H.R. 5467 IH 117th.pdf", "DoDD 3200.15 CH 1.pdf"]}, {"idvisitor": "6A4EE43B3AFF62EB", "docs_opened": 0, "searches_made": 2, "opened": [], "export": [], "favorite": []}, {"idvisitor": "71339E3B3F85E3B4", "docs_opened": 240, "searches_made": 1735, "opened": ["CI 16601.28B.pdf", "DoDD 5105.19.pdf", "CIM 16000.14A.pdf", "ATP 97.pdf", "SASC Hearing"], "export": ["AI 40.pdf", "DoDD 5137.02.pdf", "CJCSI 1805.01C.pdf", "Title 15 - Commerce and Trade.pdf", "CNSSI 4009.pdf"], "favorite": ["ARMY 2018-18 ARMY ARTIFICIAL INTELLIGENCE TASK FORCE IN SUPPORT OF THE DEPARTMENT OF DEFENSE JOINT ARTIFICIAL INTELLIGENCE CENTER", "Title 10 Armed Forces", "Department of Defense Handbook MIL-HDBK-57D Listing of Fastener Manufacturer's Identification Symbols", "EO 13822 Supporting Our Veterans During Their Transition From Uniformed Service to Civilian Life", "H.R. 922 To prohibit United States Government recognition of the Russian Federation\u2019s claim of sovereignty over Crimea, and for other purposes."]}, {"idvisitor": "721E722041A6EC2D", "docs_opened": 0, "searches_made": 6, "opened": [], "export": [], "favorite": []}, {"idvisitor": "7976822F1CDC4562", "docs_opened": 35, "searches_made": 22, "opened": ["DoDM 4140.01 Volume 9 CH 2.pdf", "DoD Data Strategy.PD[\u2026]geNumber=0", "DoD Data Strategy.PDF", "H.R. 5467 IH 117th.pdf", "H 5467 IH 117th.pdf"], "export": [], "favorite": []}, {"idvisitor": "7F06C04D59BD8360", "docs_opened": 12, "searches_made": 57, "opened": ["DoDI 5240.19 CH 2.pdf", "AFI 10-202.pdf", "ARMY DIR 2021-35.pdf", "MCO 1730.8.pdf"], "export": [], "favorite": []}, {"idvisitor": "825777C330FD094B", "docs_opened": 0, "searches_made": 28, "opened": [], "export": [], "favorite": []}, {"idvisitor": "8E0BCAF48C3D73DD", "docs_opened": 288, "searches_made": 453, "opened": ["STIG V1R1 Riverbed SteelHead CX v8 STIG  Ver 1 Release Memo.pdf", "STIG V0R0 Vendor STIG Acronym List.pdf", "STIG V0R0 CCI Process.pdf", "STIG V0R0 2016-04-21 DoD CIO Memo", "STANAG 4439.pdf"], "export": ["OO-S-256_11 Revision B Notice 2 - Validation 2006-11-01.pdf", "OO-S-256_11 Revision B Notice 3 - Validation 2020-02-05.pdf", "OO-S-256_1 Revision B Notice 3 - Validation 2020-02-04.pdf", "MIL-G-58103 Notice 3 - Validation 2010-03-23.pdf", "MIL-G-58103 Notice 4 - Validation 2014-12-23.pdf"], "favorite": ["3020.01,-Change-2  .pdf", "MIL-R-8592 Revision B Notice 2 - Validation 2019-08-22.pdf", "H.R. 5467 IH 117th.pdf"]}, {"idvisitor": "9160B3FFCCAD442E", "docs_opened": 2, "searches_made": 18, "opened": ["DoDI 5015.02 CH 1.pdf"], "export": [], "favorite": []}, {"idvisitor": "922C762749F5451C", "docs_opened": 0, "searches_made": 79, "opened": [], "export": [], "favorite": []}, {"idvisitor": "9F6BBEC48D1BBD01", "docs_opened": 0, "searches_made": 31, "opened": [], "export": [], "favorite": []}, {"idvisitor": "AB92B386244A1CE6", "docs_opened": 0, "searches_made": 2, "opened": [], "export": [], "favorite": []}, {"idvisitor": "B102CE1D5EEBAC2B", "docs_opened": 0, "searches_made": 12, "opened": [], "export": [], "favorite": []}, {"idvisitor": "B88667501EF34887", "docs_opened": 0, "searches_made": 1, "opened": [], "export": [], "favorite": []}, {"idvisitor": "CA87275CB32384EA", "docs_opened": 0, "searches_made": 4, "opened": [], "export": [], "favorite": []}, {"idvisitor": "CFE5B0C8CD296892", "docs_opened": 0, "searches_made": 5, "opened": [], "export": [], "favorite": []}, {"idvisitor": "DCF48D0B6FBA15A2", "docs_opened": 7, "searches_made": 21, "opened": ["OMBM M-16-21.pdf", "TC 3-22.50.pdf", "MCTP 3-01C.pdf", "ARMY DIR 2018-07-13.pdf"], "export": [], "favorite": ["OPNAVINST 5420.106A.pdf"]}, {"idvisitor": "DD80B9C66CADA1DA", "docs_opened": 27, "searches_made": 426, "opened": ["MIL-DTL-32541 Base Document 2015-11-30.pdf", "DoDI 1235.12 CH 1.pdf", "012  .pdf", "18-006-  .pdf"], "export": ["DoDD 5000.01.pdf", "DoDI 4151.22.pdf", "DoDI 1215.21 CH 1.pdf", "DoDI 1322.26.pdf"], "favorite": []}, {"idvisitor": "E0766A1B3314BBAD", "docs_opened": 3, "searches_made": 2, "opened": ["TM 5-642_dup1.pdf", "ARMY DIR 2020-06.pdf"], "export": [], "favorite": []}, {"idvisitor": "E33C668433DEBFCC", "docs_opened": 26, "searches_made": 136, "opened": ["NAVMC 3500.100C.pdf", "ARMY DIR 2020-06.pdf", "PGI 239.76", "DoDI 1015.10 CH 1.pdf", "MCRP 3-20B.1.pdf"], "export": [], "favorite": []}, {"idvisitor": "E879AA30341B8519", "docs_opened": 28, "searches_made": 137, "opened": ["SECNAVINST 5239.24.pdf", "DFARS SUBPART 239.76", "DFARS SUBPART 239.76.html", "MARADMIN MARADMIN CANCELLATION 011-13 CANCELLATION OF MARADMIN 103-13.html"], "export": [], "favorite": []}, {"idvisitor": "E8996EFD2D75C4CC", "docs_opened": 33, "searches_made": 137, "opened": ["MARADMIN 031-13 CANCELLATION OF MARADMIN 325-13.html", "MARADMIN 036-10 CANCELLATION OF MARADMIN 367-10.html", "MARADMIN 049-19 CANCELLATION OF MARADMIN 628-19.html", "SECNAVINST 5213.16.pdf", "MCO 6320.5.pdf"], "export": [], "favorite": ["SECNAVINST 5239.24.pdf"]}, {"idvisitor": "EA8F8BB9A8A7B95C", "docs_opened": 0, "searches_made": 5, "opened": ["EDAPDF-00AF985C0EC44D12E05400215A9BA3BA-H9222210D0016-0015-empty-01-PDS-2014-08-15.pdf"], "export": [], "favorite": []}, {"idvisitor": "EC088268C2A5CABE", "docs_opened": 7, "searches_made": 13, "opened": ["MIL-T-58100 Notice 4", "MIL-D-23222 Revision A Notice 1", "ICD 700.pdf"], "export": ["DoDI 2000.12 CH 3.pdf"], "favorite": ["SECNAVINST 3300.2C.pdf", "SECNAVINST 5239.24.pdf"]}, {"idvisitor": "EE51D5B9B57CD8A1", "docs_opened": 8, "searches_made": 42, "opened": ["OPNAVINST 5420.106A.pdf", "SECNAVINST 5800.15A.pdf", "H.R. 2812 IH 117th.pdf"], "export": [], "favorite": []}], "cards": {"unique_users": 31, "total_searches": 5543}}

gamechangerml/src/search/ranking/ltr.py

@@ -14,6 +14,7 @@
 import typing as t
 import base64
 from urllib.parse import urljoin
+from datetime import datetime,timedelta
 from gamechangerml.src.utilities import gc_web_api, es_utils
 
 
@@ -110,7 +111,9 @@ def read_mappings(
     def request_mappings(self, daysBack: int = 180):
         mappings = None
         try:
-            mappings = gcClient.getSearchMappings(daysBack=daysBack)
+            start_date=(datetime.now()-timedelta(days=daysBack)).replace(hour=0, minute=0)
+            end_date=datetime.now()
+            mappings = gcClient.getSearchMappings(start_date=start_date,end_date=end_date)
             mappings = json.loads(mappings)
             mappings = pd.DataFrame(mappings["data"])
         except Exception as e:

gamechangerml/src/utilities/gc_web_api.py

@@ -1,10 +1,11 @@
 from attr import has
+from datetime import datetime, timedelta
+
 import requests
 import os
 import base64
 import hmac, hashlib
 
-
 class GCWebClient:
     def __init__(
         self,
@@ -39,10 +40,18 @@ def getHash(self,endpoint):
         hash = base64.b64encode(h.digest())
         return hash
 
-    def getSearchMappings(self, daysBack=3):
+    def getSearchMappings(self, start_date=(datetime.now()-timedelta(days=3)).replace(hour=0, minute=0), end_date=datetime.now()):
         #endpoint needs to be hashed before adding query params since thats how the web backend calculates the hash
         endpoint = f"/api/gameChanger/admin/getSearchPdfMapping"
         hash = self.getHash(endpoint)
-        endpoint += f'?daysBack={daysBack}'
+        endpoint += f'?startDate={start_date}&endDate={end_date}'
+        r = requests.get(self.getURL + endpoint, headers=self.getHeader(hash))
+        return r.content
+
+    def getUserAggregations(self, start_date=(datetime.now()-timedelta(days=3)).replace(hour=0, minute=0), end_date=datetime.now()):
+        #endpoint needs to be hashed before adding query params since thats how the web backend calculates the hash
+        endpoint = f"/api/gameChanger/admin/getUserAggregations"
+        hash = self.getHash(endpoint)
+        endpoint += f'?startDate={start_date}&endDate={end_date}'
         r = requests.get(self.getURL + endpoint, headers=self.getHeader(hash))
         return r.content